level=info ts=2026-02-16T18:01:56.524195283Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T18:01:56.524316697Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T18:01:56.524851957Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T18:01:56.525744989Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T18:01:56.526112664Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:01:56.52618075Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.58 inf=eth0
level=info ts=2026-02-16T18:01:56.526690223Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-e00f8275
ts=2026-02-16T18:01:56.526756992Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T18:01:56.526776013Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.58 advertisePort=7946
level=debug ts=2026-02-16T18:01:56.526794704Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.58 advertisePort=7946
level=info ts=2026-02-16T18:01:56.526910857Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T18:01:56.532715944Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:01:56.532747076Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-02-16T18:01:56.534524303Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T18:01:56.534954589Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:01:56.535173492Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T18:01:56.535205994Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T18:01:56.535203911Z
level=debug ts=2026-02-16T18:01:56.535225216Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T18:01:56.535247706Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T18:01:56.535292543Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T18:01:56.535298265Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T18:01:56.535297245Z
level=debug ts=2026-02-16T18:01:56.535302759Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-02-16T18:01:56.535433424Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:01:56.535456707Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:01:56.53551Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.601029ms
level=error ts=2026-02-16T18:01:56.535523398Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T18:01:56.535545983Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T18:01:56.537646102Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:01:56.537745209Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.58 inf=eth0
ts=2026-02-16T18:01:56.539774504Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:56.5397952Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:01:56.552605132Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-02-16T18:01:56.552606485Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-02-16T18:01:56.552621804Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T18:01:56.552638704Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T18:01:56.552648238Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T18:01:56.552645464Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-02-16T18:01:56.552650808Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-02-16T18:01:56.552665942Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T18:01:56.552679381Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:01:56.552684628Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-02-16T18:01:56.55269047Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.552696824Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.552870522Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T18:01:56.552876575Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T18:01:56.552884306Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T18:01:56.552888206Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T18:01:56.552916322Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T18:01:56.552939733Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=info ts=2026-02-16T18:01:56.552943804Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T18:01:56.553025355Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:01:56.553040411Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-02-16T18:01:56.553044158Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:56.553047212Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-02-16T18:01:56.553050022Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-02-16T18:01:56.553053017Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T18:01:56.553073911Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-02-16T18:01:56.553106649Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-02-16T18:01:56.55315815Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=87.974µs errors=false
level=info ts=2026-02-16T18:01:56.553167454Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-02-16T18:01:56.553269151Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=199.781µs errors=false
level=info ts=2026-02-16T18:01:56.553278488Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T18:01:56.553287621Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=217.968µs
level=info ts=2026-02-16T18:01:56.55330943Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T18:01:56.553384463Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T18:01:56.553399934Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-02-16T18:01:56.553398528Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=7.905046076s
level=info ts=2026-02-16T18:01:56.553500874Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=info ts=2026-02-16T18:01:56.553503977Z caller=loki.go:599 msg="Loki started" startup_time=98.99772ms
level=debug ts=2026-02-16T18:01:56.553514484Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T18:01:56.553536724Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-02-16T18:01:56.553582118Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-02-16T18:01:57.898841736Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:57.898874562Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:01.463317813Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:01.463346169Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:02:01.55416538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:02:06.010812584Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:06.010842366Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:02:06.554420183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:02:11.553987582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:02:14.762568747Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:14.762594218Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:02:16.554030162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:02:20.067391497Z caller=logging.go:137 msg="GET /ready (503) 114.964µs"
level=warn ts=2026-02-16T18:02:21.554519273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:02:26.553519913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T18:02:26.553579218Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T18:02:26.553584842Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:02:26.553576247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:02:30.066262808Z caller=logging.go:137 msg="GET /ready (503) 81.581µs"
level=warn ts=2026-02-16T18:02:31.553817147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:02:31.558017097Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:02:34.067498715Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.59:7946: dial tcp 10.128.0.59:7946: connect: connection refused"
ts=2026-02-16T18:02:34.068257311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-02-16T18:02:34.06955498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.55:7946"
ts=2026-02-16T18:02:34.07174445Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.59:7946: dial tcp 10.130.0.59:7946: connect: connection refused"
ts=2026-02-16T18:02:34.07340493Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.62:7946: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-02-16T18:02:34.074563254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
level=info ts=2026-02-16T18:02:34.076142057Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=37.540589435s
level=warn ts=2026-02-16T18:02:36.554424206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:02:41.554561983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:02:43.223938375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
ts=2026-02-16T18:02:43.433073877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39334"
ts=2026-02-16T18:02:43.723091333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33224"
level=warn ts=2026-02-16T18:02:46.553667317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:02:51.553958469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:02:53.153960113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48730"
level=debug ts=2026-02-16T18:02:55.256498514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:02:56.534679273Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:02:56.535734935Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:02:56.554113652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:02:56.554140263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:02:56.554149126Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:02:56.55418284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:03:01.55434816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:03:02.734062963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:03:06.5543173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:03:11.554607384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:03:13.226333253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=warn ts=2026-02-16T18:03:16.554502129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:03:17.854762016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:03:21.554193189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:03:24.660655094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50004"
level=info ts=2026-02-16T18:03:26.554246443Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:03:26.554275403Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:03:26.554283188Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:03:26.554304522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:03:31.554287166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:03:31.586378994Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:03:34.405938666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:03:36.554239234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:03:41.554409039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:03:43.227949354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=warn ts=2026-02-16T18:03:46.554130807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:03:51.554548532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:03:53.158734883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33622"
level=debug ts=2026-02-16T18:03:56.533779853Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:03:56.535884357Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:03:56.554290769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:03:56.5543222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:03:56.554333543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:03:56.554324483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:03:57.391546847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:04:00.599109318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:04:00.599201102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:04:00.610502929Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:04:01.553991277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:04:06.553749371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:04:11.55422225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:04:13.22882427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=warn ts=2026-02-16T18:04:16.554081204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:04:18.538965049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:04:18.545410741Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:04:21.553677771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:04:24.20657571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:04:24.208342413Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:04:26.553691159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:04:26.553720218Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:04:26.553729128Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:04:26.553748789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:04:29.35371908Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:04:31.55444992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:04:36.553737277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:04:41.554395279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:04:43.229936006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=warn ts=2026-02-16T18:04:46.553760108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:04:51.55434314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:04:52.803063057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:04:52.80988625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:04:53.163249699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33164"
level=debug ts=2026-02-16T18:04:55.244836176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:04:56.534182695Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:04:56.535240717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:04:56.553671227Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:04:56.553693013Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:04:56.553701053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:04:56.553766143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:05:01.554612069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:05:06.553786644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:05:08.229118503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:05:11.554319609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:05:13.232275983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=warn ts=2026-02-16T18:05:16.553797462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:05:21.553637868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:05:24.109114952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:05:24.669379843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46560"
level=info ts=2026-02-16T18:05:26.553982742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:05:26.554014605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:05:26.554023247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:05:26.554022618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:05:30.882459432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51776"
level=warn ts=2026-02-16T18:05:31.554334857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:05:36.554434306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:05:41.184798253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:05:41.193383498Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:05:41.554189556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:05:43.235169454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=warn ts=2026-02-16T18:05:46.553612257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:05:50.177656768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:05:51.556174364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:05:53.168388145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:54204"
level=info ts=2026-02-16T18:05:56.53515929Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:05:56.535186364Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:05:56.55350663Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:05:56.553529909Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:05:56.553540823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:05:56.553565107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:06:01.553869612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:06:06.554515664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:06:11.553872822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:06:13.23825345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:06:16.2070054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:06:16.554520792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:06:20.262742567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:06:21.55461749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:06:23.169751663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46060"
ts=2026-02-16T18:06:24.67189748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41802"
level=info ts=2026-02-16T18:06:26.553526447Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:06:26.553559046Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:06:26.55356881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:06:26.553581301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:06:29.60543501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:06:29.613459529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:06:31.553661503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:06:36.554032939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:06:41.554054259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:06:43.239982422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=warn ts=2026-02-16T18:06:46.554153371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:06:51.311130746Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:06:51.553583405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:06:56.534365487Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:06:56.535420137Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:06:56.553759626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:06:56.553775066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:06:56.55379048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:06:56.553798532Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:06:56.553812774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:06:56.553936099Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-02-16T18:06:56.554288798Z caller=checkpoint.go:378 msg="writing series" size="276 kB"
level=debug ts=2026-02-16T18:07:01.056440944Z caller=checkpoint.go:378 msg="writing series" size="3.5 MB"
level=debug ts=2026-02-16T18:07:01.05646669Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.8 MB" series=2
level=warn ts=2026-02-16T18:07:01.554224023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:05.554902106Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-02-16T18:07:06.554013985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:10.054413004Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=warn ts=2026-02-16T18:07:11.55391479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:11.717528599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:07:13.241925454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:07:14.554271332Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-02-16T18:07:16.187585628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:07:16.553712192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:19.054796243Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-02-16T18:07:21.553986151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:23.554815182Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-02-16T18:07:26.321508963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:07:26.330762468Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:07:26.554361504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:07:26.554376493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:26.554392199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:07:26.554405233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:07:28.054741455Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
ts=2026-02-16T18:07:30.894557445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56508"
level=warn ts=2026-02-16T18:07:31.554359991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:32.554759983Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T18:07:36.553923521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:37.054540773Z caller=checkpoint.go:378 msg="writing series" size="330 kB"
level=warn ts=2026-02-16T18:07:41.5536801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:41.554726529Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-02-16T18:07:43.244523845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:07:46.054645614Z caller=checkpoint.go:378 msg="writing series" size="310 kB"
level=debug ts=2026-02-16T18:07:46.054671508Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-02-16T18:07:46.553910754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:50.554283695Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-02-16T18:07:51.554314382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:52.004986437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:07:52.005071013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:07:52.00998292Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:07:52.30608061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:07:52.309236955Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:07:52.605309124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="25 kB" avg_uncomp="25 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:07:52.608637478Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T18:07:53.177942744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43148"
level=debug ts=2026-02-16T18:07:55.054427977Z caller=checkpoint.go:378 msg="writing series" size="345 kB"
level=info ts=2026-02-16T18:07:56.53518619Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:07:56.535212186Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:07:56.553580706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:07:56.553605017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:07:56.553626642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:07:56.553633601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:07:59.554447393Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=warn ts=2026-02-16T18:08:01.553795197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:08:02.808751255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:08:02.811989749Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:08:04.054934984Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T18:08:06.553912423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:08:06.655249565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:08:06.65881811Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:08:07.027589194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:08:08.555135225Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T18:08:11.451627409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:08:11.455189883Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:08:11.55442745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:08:13.054333234Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
ts=2026-02-16T18:08:13.247049222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=warn ts=2026-02-16T18:08:16.554271538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:08:17.55512392Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T18:08:18.668394795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:08:18.668451925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T18:08:18.672203839Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:08:18.691275024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:08:18.69814908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:08:18.969105153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:08:18.972413199Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:08:19.269290792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:08:19.273568809Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:08:19.568652559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:08:19.572024594Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:08:19.868863233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:08:19.872183704Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:08:20.16915282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="50 kB" avg_uncomp="50 kB" idle=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:08:20.172461096Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:08:20.469289658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="54 kB" avg_uncomp="54 kB" idle=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:08:20.47282316Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:08:20.768861609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="274 B" avg_comp="274 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:08:20.7722994Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:08:21.069151949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="7.4 kB" avg_comp="7.4 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" idle=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:08:21.072541949Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:08:21.368498287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="8.1 kB" avg_comp="8.1 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:08:21.371832417Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:08:21.554589224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:08:21.669407833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:08:21.672794233Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:08:21.968843972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="46 kB" avg_comp="46 kB" total_uncomp="46 kB" avg_uncomp="46 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:08:21.97390528Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:08:22.055153985Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:08:22.268506548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="5.0 kB" avg_comp="5.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:08:22.272062458Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:08:23.179656278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59010"
level=info ts=2026-02-16T18:08:25.730417018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:08:25.733881451Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:08:26.553508164Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:08:26.553542725Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:08:26.553551319Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:08:26.553598356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:08:26.554627884Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-02-16T18:08:31.0546682Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-02-16T18:08:31.553926654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:08:31.806444187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:08:31.809780607Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:08:32.324615581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:08:32.328389715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:08:32.410494314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:08:32.414083429Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:08:32.812316073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:08:32.815924934Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:08:33.98074376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:08:33.984109574Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:08:34.709912682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:08:34.713421419Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:08:35.554456096Z caller=checkpoint.go:378 msg="writing series" size="318 kB"
level=info ts=2026-02-16T18:08:35.564433352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:08:35.567705581Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:08:35.730308858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:08:35.733620429Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:08:36.002265399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:08:36.005509982Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T18:08:36.554186096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:08:36.843592155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:08:36.847116883Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:08:38.184380779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:08:38.187739905Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:08:38.463472447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:08:38.469177639Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:08:38.99785141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:08:39.001654457Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:08:40.054795507Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=info ts=2026-02-16T18:08:41.074757389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:08:41.078553575Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:08:41.148902074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:08:41.152529966Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:08:41.554500616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:08:43.250239562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:08:44.554805297Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-02-16T18:08:44.788556411Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:08:46.553942399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:08:49.054658523Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-02-16T18:08:51.553815919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:08:53.181309941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39212"
level=debug ts=2026-02-16T18:08:53.554924066Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=debug ts=2026-02-16T18:08:56.533704425Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:08:56.535863378Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:08:56.554196829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:08:56.554224231Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:08:56.554234834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:08:56.554255053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:08:57.336259119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:08:57.339467409Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:08:58.055190827Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=debug ts=2026-02-16T18:08:58.055214147Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=16
level=info ts=2026-02-16T18:08:58.545930313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:08:58.549249568Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:09:00.63381817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:09:00.637294509Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:09:00.776738437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:09:00.780337107Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-02-16T18:09:00.903590004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49320"
level=warn ts=2026-02-16T18:09:01.553631183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:02.55450356Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T18:09:04.579428649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:09:04.581233528Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:09:04.594328533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:09:04.597721724Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:09:04.665953143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:09:04.669267554Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:09:06.554445838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:07.054623137Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=info ts=2026-02-16T18:09:07.279073211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:09:07.282488588Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:09:08.066716455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:09:08.070421208Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:09:09.819110392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:09:09.822931844Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:09:10.012547782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:09:10.016163883Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:09:10.748111082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:09:10.751846602Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:09:11.43982412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:09:11.443639845Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T18:09:11.553967186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:11.555024349Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
ts=2026-02-16T18:09:13.253823883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:09:13.900318299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:09:13.906865226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:09:15.274136395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:09:16.012398447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:09:16.015854836Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:09:16.055150021Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:09:16.553585364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:09:17.295980705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:09:17.299686595Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T18:09:20.554615296Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:09:21.5232129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:09:21.527201169Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:09:21.553743959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:09:23.182774291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40206"
level=info ts=2026-02-16T18:09:23.536448978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:09:23.542039144Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:09:24.682431058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54844"
level=debug ts=2026-02-16T18:09:25.054793103Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
level=warn ts=2026-02-16T18:09:26.553659557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:09:26.553682113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:09:26.55370488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:09:26.553713278Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:09:29.554939233Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-02-16T18:09:31.554374207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:34.055135044Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
level=warn ts=2026-02-16T18:09:36.553718516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:38.554796914Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:09:41.554252963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:42.732458795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:09:43.054427235Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
ts=2026-02-16T18:09:43.256255231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:09:43.256526881Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:09:46.554360762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:09:46.842049337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:09:46.845583738Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:09:47.554780319Z caller=checkpoint.go:378 msg="writing series" size="714 B"
level=info ts=2026-02-16T18:09:50.538038217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:09:50.541470408Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:09:50.869355428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:09:50.872748674Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:09:51.553721204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:52.054923411Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=info ts=2026-02-16T18:09:52.289444176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:09:52.292987662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:09:52.402372355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:09:52.405927475Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:09:53.183784937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43280"
level=info ts=2026-02-16T18:09:54.36430675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:09:54.367851466Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:09:55.336532564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:09:55.339876384Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:09:55.899305658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:09:55.902772025Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:09:56.065331886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:09:56.068691484Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:09:56.534054228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:09:56.535104475Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:09:56.553502539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:09:56.553528059Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:09:56.553537694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:09:56.553607397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:09:56.55454786Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T18:10:00.087445353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:10:00.090871741Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T18:10:00.909583355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50390"
level=debug ts=2026-02-16T18:10:01.055082277Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=debug ts=2026-02-16T18:10:01.055108647Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-02-16T18:10:01.554433207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:10:02.162272714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:10:02.165798082Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:10:02.698899638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:10:02.705344551Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:10:03.168076427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:10:03.171718731Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:10:04.26135844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:10:04.264959255Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:10:05.555137715Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=info ts=2026-02-16T18:10:06.16088694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:10:06.165123326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T18:10:06.553730726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:10:08.244664917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:10:08.248661801Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:10:10.054564808Z caller=checkpoint.go:378 msg="writing series" size="290 kB"
level=debug ts=2026-02-16T18:10:10.067771583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:10:11.554111162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:10:12.573465841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:10:12.578342533Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:10:13.257587021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:10:14.554313855Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-02-16T18:10:16.554269179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:19.055961123Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T18:10:19.055984223Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-02-16T18:10:21.553937237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:23.55506757Z caller=checkpoint.go:378 msg="writing series" size="508 B"
level=info ts=2026-02-16T18:10:26.554251441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:10:26.554281817Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:10:26.554290044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:10:26.554303375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:28.054692757Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=debug ts=2026-02-16T18:10:30.860559363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:10:30.912946288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43980"
level=info ts=2026-02-16T18:10:31.016815048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:10:31.02086059Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:10:31.554455011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:32.554743068Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:10:34.718010021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:10:34.722375125Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:10:36.554143765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:37.054153826Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T18:10:40.839012589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:10:40.842794448Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:10:41.553961778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:10:41.555034424Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
ts=2026-02-16T18:10:43.259401243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:10:44.001220351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:10:44.001227161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="23 kB" avg_uncomp="23 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:10:44.0054092Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:10:44.021499653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:10:44.024889581Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:10:44.111063242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:10:44.11514605Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:10:44.302154873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="294 B" avg_uncomp="294 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:10:44.3058608Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:10:44.456369142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:10:44.460185296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:10:44.601493971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="394 kB" avg_uncomp="394 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:10:44.605824158Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:10:44.952926451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:10:44.956516192Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:10:45.301236448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:10:45.3046637Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:10:46.055030328Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-02-16T18:10:46.320768539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:10:46.324547485Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:10:46.554397875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:10:48.646889632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:10:48.650173622Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:10:50.554876426Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-02-16T18:10:51.280999374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:10:51.284749803Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:10:51.554517013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:10:52.590223188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:10:52.593860471Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T18:10:53.188156884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41878"
level=info ts=2026-02-16T18:10:53.655340092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:10:53.658712075Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:10:54.010534769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:10:54.014532624Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:10:55.054630407Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T18:10:56.182763375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:10:56.186345265Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:10:56.534462313Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:10:56.535511505Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:10:56.554396681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:10:56.554431331Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:10:56.554440841Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:10:56.554461442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:10:58.450457731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:10:58.453973864Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:10:58.719682903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:10:58.727155311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:10:58.727181763Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:10:58.727210033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:10:58.728464864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:10:58.733268875Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:10:58.733272669Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:10:59.554719066Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T18:11:01.554598166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:02.126832581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:11:02.130609228Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:11:04.054539621Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=info ts=2026-02-16T18:11:04.225275366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:11:04.230784627Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:11:06.553779003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:08.55501485Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T18:11:11.040065391Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:11:11.554627236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:11.825885671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:11:13.05481161Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
ts=2026-02-16T18:11:13.2607259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:11:14.493612402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:11:14.501464373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:11:16.553804514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:17.554758955Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T18:11:21.554320897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:22.05471382Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-02-16T18:11:23.188987628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60328"
level=info ts=2026-02-16T18:11:24.611770265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:11:24.616800136Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:11:25.125176248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:11:25.129379817Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:11:25.231696078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:11:25.235759103Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:11:26.554448708Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:11:26.554506468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:11:26.554503494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:26.55451787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:11:26.554512421Z caller=checkpoint.go:390 msg="flushing series" totalSize="438 kB" series=14
level=info ts=2026-02-16T18:11:26.559833759Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-02-16T18:11:26.56329511Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009221879s
level=info ts=2026-02-16T18:11:27.942148417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:11:27.946940564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:11:28.535753731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:11:28.5405443Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:11:28.702744355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:11:28.707377395Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:11:31.12194315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:11:31.125828919Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:11:31.55445824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:31.880079155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:11:31.883964545Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:11:33.469492375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:11:33.473341733Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:11:36.498516357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:11:36.502627982Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:11:36.553899016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:37.654869106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:11:37.658357099Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:11:38.29811062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:11:38.30171136Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:11:38.606942084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:11:38.611128856Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:11:38.661466393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:11:38.669739532Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:11:39.149240474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:11:39.153175468Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:11:39.871381366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:11:39.874862279Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:11:41.050958079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:11:41.054605992Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:11:41.554103782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:42.464152793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:11:43.262226549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:11:43.776733653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:11:43.781211121Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:11:46.554316602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:11:51.554056051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:11:52.375861099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:11:52.379534634Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:11:52.523806285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:11:52.527084482Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T18:11:52.609167511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:11:56.533881068Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:11:56.536013449Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:11:56.553353553Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:11:56.553494075Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T18:11:56.553514955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:11:56.553542578Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:11:56.553559601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:11:56.553581903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:11:56.553757077Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=debug ts=2026-02-16T18:12:00.414038906Z caller=checkpoint.go:378 msg="writing series" size="4.3 MB"
level=debug ts=2026-02-16T18:12:00.414064588Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.5 MB" series=2
level=warn ts=2026-02-16T18:12:01.55433272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:04.268985768Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:12:06.554572623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:12:07.479049444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:12:07.485655997Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:12:08.125561859Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-02-16T18:12:11.5536051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:11.983120432Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=info ts=2026-02-16T18:12:12.767262071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:12:12.772169805Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:12:13.264765589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:12:14.106564709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:12:14.110160768Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:12:15.426899219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:12:15.432790825Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:12:15.840032979Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=warn ts=2026-02-16T18:12:16.554023515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:12:17.901003507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:12:17.905210921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:12:18.326513613Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:12:19.466692154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:12:19.47010445Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:12:19.697004299Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-02-16T18:12:20.506825431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:12:20.510794604Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T18:12:21.553608911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:12:21.793703376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:12:21.800042709Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:12:22.346680626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:12:22.350155442Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:12:23.191360308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37780"
level=debug ts=2026-02-16T18:12:23.554800271Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=info ts=2026-02-16T18:12:24.207610962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:12:24.211443256Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:12:24.687778961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:12:24.691087171Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:12:24.708160408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:12:24.711377197Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:12:25.397382065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:12:25.400921038Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:12:26.554427635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:12:26.554460084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:12:26.554469327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:12:26.554522244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:27.411645762Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=info ts=2026-02-16T18:12:28.314279621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:12:28.317740461Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:12:29.006851963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:12:29.010016714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:12:30.84831849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:12:30.851861074Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:12:31.26844744Z caller=checkpoint.go:378 msg="writing series" size="506 kB"
level=debug ts=2026-02-16T18:12:31.268472169Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-02-16T18:12:31.341819429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:12:31.345704908Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:12:31.452974251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:12:31.45656412Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T18:12:31.554145328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:35.125554402Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T18:12:36.553915758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:38.982411122Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:12:41.232598214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:12:41.23600342Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:12:41.553822528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:12:42.48629487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:12:42.489684587Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T18:12:42.84021Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-02-16T18:12:43.113766166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:12:43.115635776Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:12:43.266431612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=warn ts=2026-02-16T18:12:46.553559232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:46.697545842Z caller=checkpoint.go:378 msg="writing series" size="369 kB"
level=debug ts=2026-02-16T18:12:50.554060188Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-02-16T18:12:50.994386398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:12:51.554058252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:12:54.411738763Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
level=info ts=2026-02-16T18:12:55.963974343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:12:55.970911965Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:12:56.535043382Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:12:56.535296918Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:12:56.553692896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:12:56.553733742Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:12:56.55374648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:12:56.553753871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:12:57.245119506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:12:57.249193734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:12:58.268981984Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
ts=2026-02-16T18:13:00.928959195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38598"
level=warn ts=2026-02-16T18:13:01.554187411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:02.12521707Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T18:13:02.96634948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:13:02.970567934Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:13:05.681407935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:13:05.685221987Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:13:05.983173014Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-02-16T18:13:06.107495996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:13:06.111691132Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T18:13:06.553977479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:08.129346319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:13:08.133979014Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:13:09.103718367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:13:09.115697089Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:13:09.840056435Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:13:11.233698654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:13:11.237080597Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:13:11.55443065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:11.930033415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:13:11.933716794Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:13:13.268308137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:13:13.697197567Z caller=checkpoint.go:378 msg="writing series" size="1.6 kB"
level=info ts=2026-02-16T18:13:15.166795213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:13:15.173896158Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:13:15.185944918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:13:15.189210233Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:13:15.867277918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:13:15.871503694Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:13:16.554307269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:17.554274896Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=info ts=2026-02-16T18:13:17.981429793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:13:17.98578665Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:13:21.411666142Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-02-16T18:13:21.554372487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:23.057243178Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:13:23.659213736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:13:23.663644424Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:13:24.555277774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:13:24.559651301Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:13:24.822194073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:13:24.826239781Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:13:24.92151587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:13:24.925250358Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T18:13:25.268176914Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:13:26.553859947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:13:26.553889829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:13:26.553898156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:13:26.553917762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:28.752891067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:13:28.756832129Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:13:29.125975142Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-02-16T18:13:29.453114184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:13:30.080143647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:13:30.083880104Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T18:13:30.932567909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57628"
level=warn ts=2026-02-16T18:13:31.55374784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:31.718717703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:13:31.722494173Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:13:32.982606306Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T18:13:36.554159069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:36.844129928Z caller=checkpoint.go:378 msg="writing series" size="7.3 MB"
level=debug ts=2026-02-16T18:13:36.8441613Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.3 MB" series=17
level=debug ts=2026-02-16T18:13:40.697592993Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=warn ts=2026-02-16T18:13:41.554470037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:41.670868822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:13:41.675398272Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:13:41.67542453Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:13:41.675448743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:13:41.678988116Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:13:42.008440784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:13:42.014051912Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:13:43.269456474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:13:44.554387528Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-02-16T18:13:46.553626182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:13:48.411263Z caller=checkpoint.go:378 msg="writing series" size="330 kB"
level=info ts=2026-02-16T18:13:48.436437636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:13:48.441418931Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:13:51.329509023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:13:51.333631775Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:13:51.554114234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:51.567265892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:13:51.572062245Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:13:52.268659115Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T18:13:52.32792713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:13:52.332551543Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:13:52.332580567Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:13:52.332606246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:13:52.336208237Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:13:54.677546952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:13:55.248693216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:13:55.255950628Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:13:56.125291821Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-02-16T18:13:56.534847029Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:13:56.535945087Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:13:56.554228424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:13:56.554249566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:13:56.554258499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:13:56.554263106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:13:57.700725763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:13:57.705780127Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:13:58.139114856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:13:58.143119255Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:13:58.824917321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:13:58.828828892Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:13:59.486613727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:13:59.490077361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:13:59.98264538Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T18:14:00.47228618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:14:00.475842484Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:14:01.55390814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:03.839532023Z caller=checkpoint.go:378 msg="writing series" size="160 kB"
level=info ts=2026-02-16T18:14:03.866533324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:14:03.870016954Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:14:06.554035351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:07.697488103Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:14:07.851109296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:14:07.854585529Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:14:08.243796801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:14:08.247092817Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T18:14:10.261309188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:14:10.266011409Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T18:14:10.670263414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:14:10.674454335Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:14:10.674497134Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:10.674521804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:14:10.677292457Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:14:11.553758557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:11.553876213Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-02-16T18:14:13.10445214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:14:13.107991115Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:14:13.270839837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:14:15.411424855Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-02-16T18:14:15.411456898Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-02-16T18:14:16.553621452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:17.727704612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:14:17.732000983Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:14:19.268354552Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:14:21.554330968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:21.748858531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:14:21.75307376Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:14:22.968919839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:14:22.972643054Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:14:23.126015999Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
ts=2026-02-16T18:14:23.203085233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48066"
level=info ts=2026-02-16T18:14:24.372550247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:14:24.37717594Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:14:24.415259632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:14:24.41896018Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:14:24.956083832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:14:24.95973713Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:14:26.554430098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:14:26.554461216Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:14:26.554469814Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:14:26.554457894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:26.563572873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:14:26.866339805Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:14:26.983356094Z caller=checkpoint.go:378 msg="writing series" size="310 kB"
level=info ts=2026-02-16T18:14:30.498497833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:14:30.502710534Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:14:30.840175504Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-02-16T18:14:31.554013995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:34.01694778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:14:34.020264343Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:14:34.697533937Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-02-16T18:14:36.553789952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:38.554515328Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:14:40.872299744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:14:40.876318343Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T18:14:40.876342839Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:40.876367321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:14:40.879408963Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:14:41.527452476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:14:41.531584953Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:14:41.531614584Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:41.531645773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:14:41.535001576Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:14:41.554221935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:41.807295417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:14:41.811010608Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:14:42.116278749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:14:42.119708236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:14:42.119729029Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:42.119752282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:14:42.122498879Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:14:42.411041369Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:14:43.05977508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:14:43.063096808Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:14:43.063116976Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
ts=2026-02-16T18:14:43.272067939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:14:43.364180758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:14:43.367881974Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:14:44.53814342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:14:44.545388354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:14:46.269029665Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-02-16T18:14:46.553690622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:50.125154655Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T18:14:50.940134435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:14:50.944014815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:14:50.944038475Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:50.94406241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:14:50.947324499Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T18:14:51.553739205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:14:52.445009103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:14:52.451338926Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:14:53.205077203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33624"
level=info ts=2026-02-16T18:14:53.505975465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:14:53.509987456Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:14:53.528392229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:14:53.528453888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:14:53.534458482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:14:53.984313046Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
ts=2026-02-16T18:14:54.715193616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58862"
level=info ts=2026-02-16T18:14:54.727714739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T18:14:54.733144988Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c679e9f14%3A19c679e9f15%3Aee73a40c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T18:14:54.733181803Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:54.733220214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:14:54.745167096Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:14:55.349105585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:14:55.353493842Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:14:55.500736744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:14:55.505100164Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:14:55.505125349Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:14:55.50515014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:14:55.508835654Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:14:55.517924927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:14:55.522310734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:14:56.53445332Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:14:56.535458584Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:14:56.553847698Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:14:56.55387614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:14:56.553887813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:14:56.553902282Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:14:56.856550846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:14:56.864669349Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:14:57.840649376Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=info ts=2026-02-16T18:15:00.524819396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:15:00.530913434Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:15:01.554072503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:01.697204995Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=debug ts=2026-02-16T18:15:01.697236034Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=12
level=info ts=2026-02-16T18:15:04.349543087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:15:04.353900594Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:15:04.353929869Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:15:04.35395638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:15:04.358163849Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:15:05.227029173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:05.231421869Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:05.231451777Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:15:05.231504047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:05.235096766Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:15:05.400618661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:15:05.405689802Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:15:05.55443069Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-02-16T18:15:06.554090805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:06.832844017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:15:06.838242368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:15:08.045233563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T18:15:08.048908182Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c679e9c65%3A19c679e9f15%3Ae5c9e86a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T18:15:08.048929073Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:15:08.048950409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="99 kB" avg_uncomp="99 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:15:08.051932148Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:15:08.985813305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:15:09.411282258Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=info ts=2026-02-16T18:15:10.271090006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:10.274702574Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:10.274736783Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:15:10.274771104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:10.277946846Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:15:10.488462788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:15:10.492061202Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:15:10.492084542Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:15:10.575539875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:15:10.579050021Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:15:11.554350639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:12.057440369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:15:12.065941559Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T18:15:13.268393014Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
ts=2026-02-16T18:15:13.274625063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:15:13.837938484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:15:13.841739635Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:15:16.459872512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:15:16.463788777Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:15:16.554506201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:17.126178033Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:15:18.029800755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:15:18.033798016Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:15:18.619308881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:15:18.622778615Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:15:18.622805898Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:15:18.622828764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:15:18.625813657Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:15:19.387066057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:15:19.391467674Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:15:20.98347247Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T18:15:20.983521723Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=5
level=warn ts=2026-02-16T18:15:21.554614997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:22.601804707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:15:22.606759731Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:15:22.769307429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:22.773042732Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:15:23.243409972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:23.24721047Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:15:24.317230955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:15:24.320994447Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:15:24.675206127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:15:24.679111014Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:15:24.839755745Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-02-16T18:15:26.554442029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:15:26.554469229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:15:26.554490159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:15:26.554515712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:27.942737328Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:15:28.166302848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:15:28.169757437Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:15:28.696627019Z caller=checkpoint.go:378 msg="writing series" size="508 B"
level=info ts=2026-02-16T18:15:28.826524523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:15:28.834237145Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:15:30.3669193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:15:30.367995025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:15:30.370336135Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:15:30.37246577Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:15:31.553808222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:32.554460498Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=info ts=2026-02-16T18:15:32.811891103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:15:32.817199703Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:15:33.59005688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:15:33.594654384Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:15:33.753145397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:15:33.757051423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:15:33.887319125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:15:33.891295402Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:15:34.100085434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" idle=1 full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:15:34.105216738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:15:36.411189476Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-02-16T18:15:36.554316755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:40.268460924Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T18:15:40.696907905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:15:40.700766351Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:15:41.554374959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:15:42.076974006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:15:42.081702129Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:15:43.277243847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:15:44.125297856Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T18:15:45.8993895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:15:45.905667979Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:15:46.553642983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:47.982641326Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T18:15:48.926348033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:15:48.933829532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T18:15:51.554209721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:51.839850509Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:15:51.896058428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:15:51.900018145Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:15:53.945101683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:15:53.949272406Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:15:54.720560372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60956"
level=info ts=2026-02-16T18:15:55.274061388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:15:55.277685363Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:15:55.696952766Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T18:15:56.534196689Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:15:56.535271907Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:15:56.536610886Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T18:15:56.536656943Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968072 n=1
level=debug ts=2026-02-16T18:15:56.536701591Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968072 n=1
level=info ts=2026-02-16T18:15:56.55370607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:15:56.553728941Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:15:56.553737041Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:15:56.553769013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:15:58.980818927Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:15:58.980844491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="340 B" avg_uncomp="340 B" idle=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:15:58.984735205Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:15:59.554557751Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:16:00.731119677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:16:00.735677272Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:16:01.34832345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:16:01.352240206Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:16:01.554187174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:02.582231668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:16:02.585878241Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:16:03.410890282Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=debug ts=2026-02-16T18:16:06.029754524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:16:06.55446055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:07.268805678Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:16:10.51701447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:16:10.520910608Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:16:10.936649512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:16:10.939217861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:16:10.940661866Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:16:10.942566489Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:16:11.126161489Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=warn ts=2026-02-16T18:16:11.554554409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:16:13.278622153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:16:13.690777813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:16:13.69618174Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:16:14.733462367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:16:14.737508001Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:16:14.983212948Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-02-16T18:16:16.554311177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:18.84033092Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:16:20.420601381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:16:20.424467219Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:16:21.554471268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:21.569587932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:16:21.5732895Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:16:21.73989745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:16:21.743650778Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:16:22.18114877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:16:22.18510198Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:16:22.2017101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:16:22.220162619Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:16:22.697677045Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
ts=2026-02-16T18:16:23.213617794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35638"
level=info ts=2026-02-16T18:16:24.245636494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:16:24.253509376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:16:26.55351023Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:16:26.553549934Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:16:26.553563135Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:16:26.553573216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:26.554650924Z caller=checkpoint.go:390 msg="flushing series" totalSize="665 kB" series=16
level=info ts=2026-02-16T18:16:26.714231907Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T18:16:26.719580902Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.16594354s
level=info ts=2026-02-16T18:16:26.820760715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:16:26.826004823Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:16:29.942252803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:16:29.94569302Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:16:30.952586053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45138"
level=info ts=2026-02-16T18:16:31.523569891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:16:31.5320651Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T18:16:31.554447099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:33.785846416Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:16:36.175508653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:16:36.179892218Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:16:36.405187014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:16:36.419038743Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:16:36.553560596Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:36.750694388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:16:36.754343986Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:16:38.220357026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:16:38.22665546Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:16:41.554401689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:16:43.280252939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:16:46.198713795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:16:46.211739974Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:16:46.554129688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:16:50.223588311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:16:50.225632584Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:16:51.553606067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:51.764602911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:16:51.768147576Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:16:52.109075614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:16:52.117877272Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:16:53.199556684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:16:53.215701551Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:16:55.042303325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:16:55.046128982Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:16:56.534514344Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:16:56.535511912Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:16:56.553843755Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:16:56.553870751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:16:56.553905498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:16:56.553915581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:16:56.553908252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:16:56.553984933Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-02-16T18:16:56.554247405Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=info ts=2026-02-16T18:16:56.908281306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:16:56.920608355Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:16:59.271695137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:16:59.275220239Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:17:00.415125107Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB"
level=debug ts=2026-02-16T18:17:00.415158363Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.8 MB" series=2
level=warn ts=2026-02-16T18:17:01.554151633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:03.366460948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:17:03.370795104Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:17:03.370818709Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:17:03.370842091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:17:03.37369214Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:17:04.0007448Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:17:04.268823061Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:17:05.157457735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:17:05.161217647Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:17:06.553737471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:06.643992165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:17:06.649190294Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:17:08.126012082Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-02-16T18:17:11.554129269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:11.98361073Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=info ts=2026-02-16T18:17:12.612064497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:17:12.616329589Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:17:12.834172789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:17:12.837850118Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T18:17:13.28134517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:17:15.391613052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:17:15.40125602Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T18:17:15.840625244Z caller=checkpoint.go:378 msg="writing series" size="432 kB"
level=warn ts=2026-02-16T18:17:16.553577386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:16.571650425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:17:16.575704219Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:17:16.963791174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:17:16.967351456Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:17:17.101519019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:17:17.104742723Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:17:18.432686109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:18.439943383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:18.439968555Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:17:18.439997026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:18.446343854Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:17:19.195688997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:17:19.198995213Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:17:19.69758773Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-02-16T18:17:20.350746955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:17:20.35430091Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:17:20.379379576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:17:20.384806471Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:17:20.640762364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:17:20.644662884Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:17:20.64469087Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:17:20.64472259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:17:20.647821106Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:17:21.554347966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:17:23.219850181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51546"
level=debug ts=2026-02-16T18:17:23.554685273Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=debug ts=2026-02-16T18:17:23.554711317Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-02-16T18:17:26.553503767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:17:26.553535547Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:17:26.553544457Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:17:26.553586448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:27.411617512Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-02-16T18:17:30.277948965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:17:30.283686727Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:17:30.283710392Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:17:30.283737328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:17:30.286765474Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:17:30.791390398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:17:30.797774736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:17:31.269371264Z caller=checkpoint.go:378 msg="writing series" size="553 kB"
level=info ts=2026-02-16T18:17:31.536237321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:17:31.539830823Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:17:31.553932464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:33.192245384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:17:33.196203531Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:17:33.932697119Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:17:35.12567939Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:17:35.130891089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:17:35.13434521Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:17:36.554561991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:37.446859319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:17:37.450453561Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:17:38.983215695Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T18:17:39.10659541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:17:39.109931409Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:17:41.248033168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:17:41.251643235Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:17:41.4481628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:17:41.554233177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:42.840328359Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
ts=2026-02-16T18:17:43.284008003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:17:45.169469256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:17:45.17329871Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:17:46.553671977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:46.697782623Z caller=checkpoint.go:378 msg="writing series" size="478 kB"
level=debug ts=2026-02-16T18:17:46.697814144Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-02-16T18:17:48.042081032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:17:48.046234354Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:17:50.210153684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:17:50.214124932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:17:50.554311216Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:17:51.195079292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:17:51.198602674Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:17:51.553860331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:17:52.047312821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:17:52.050962275Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:17:53.221267429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57166"
level=info ts=2026-02-16T18:17:54.178707641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:17:54.182449094Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:17:54.412375416Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=info ts=2026-02-16T18:17:54.974153292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:17:54.977894776Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:17:55.466264169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:17:55.470176035Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:17:56.534336317Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:17:56.53539172Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:17:56.553729905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:17:56.553756178Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:17:56.553765036Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:17:56.553794339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:17:58.269372609Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:18:01.293385409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:18:01.297796719Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:18:01.554328299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:02.126266429Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-02-16T18:18:02.282774086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:18:02.286240287Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:18:04.289635575Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:18:04.289711171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="35 kB" avg_uncomp="35 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:18:04.29368868Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:18:05.612067634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:18:05.61572251Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:18:05.745100752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:18:05.752428417Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T18:18:05.983113358Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-02-16T18:18:06.553840452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:06.762364523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:18:06.7660608Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:18:07.776447748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="9.8 kB" avg_uncomp="9.8 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:18:07.78127412Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:18:09.840083062Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-02-16T18:18:10.068890944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:18:10.076185561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:18:10.787505889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:18:10.79130924Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:18:11.554457171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:12.224046632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:18:12.230124808Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:18:13.286873689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:18:13.697563079Z caller=checkpoint.go:378 msg="writing series" size="1.9 kB"
level=info ts=2026-02-16T18:18:13.909156399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:18:13.914308913Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:18:16.55369636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:17.256203593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:18:17.25993138Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:18:17.555011945Z caller=checkpoint.go:378 msg="writing series" size="859 B"
level=debug ts=2026-02-16T18:18:18.328750532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:18:19.25640477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:18:19.260389446Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:18:19.814168898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:18:19.818008599Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:18:21.411628437Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-02-16T18:18:21.554003196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:18:23.222777615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53514"
level=debug ts=2026-02-16T18:18:25.269465884Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:18:26.554196013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:18:26.554194499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:26.554229352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:18:26.554239931Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:18:27.168769467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="439 B" avg_comp="439 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:18:27.173277751Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:18:29.126357814Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-02-16T18:18:30.965671219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37230"
level=warn ts=2026-02-16T18:18:31.554301602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:32.98382551Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-02-16T18:18:33.324672403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:18:33.330654951Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:18:33.687777466Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:18:33.749093596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:18:33.753334654Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:18:33.813430445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:18:33.817430206Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:18:34.53077206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:18:34.534632228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:18:35.843729973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:18:35.847376682Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:18:36.554614415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:36.846928898Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-02-16T18:18:36.846958115Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=13
level=debug ts=2026-02-16T18:18:40.698005583Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=warn ts=2026-02-16T18:18:41.554468393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:42.408777869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:18:42.416148926Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:18:43.289027293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:18:44.390051159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:18:44.391759046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:18:44.393891889Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:18:44.393908133Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:18:44.554906945Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-02-16T18:18:45.231537147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:18:45.235082353Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:18:45.280179475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:18:45.28391631Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:18:46.553622407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:18:48.41211088Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=info ts=2026-02-16T18:18:50.805296533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:18:50.809300743Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:18:51.409019004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:18:51.412856422Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:18:51.554567653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:52.077069373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="503 kB" avg_uncomp="251 kB" idle=1 full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:18:52.080874664Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:18:52.26873866Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
ts=2026-02-16T18:18:53.224611862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60470"
level=debug ts=2026-02-16T18:18:53.516659472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:18:54.646914819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:18:54.650260346Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:18:54.730723194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55852"
level=info ts=2026-02-16T18:18:54.952947114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:18:54.959970912Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T18:18:56.126289794Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=debug ts=2026-02-16T18:18:56.534371161Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:18:56.535421021Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:18:56.55373898Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:18:56.553763784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:18:56.553772417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:18:56.553801059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:18:56.887409899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:18:56.892316025Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:18:59.799616514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:18:59.806107535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:18:59.983626476Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:19:00.416726499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:19:00.420249365Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T18:19:01.554595535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:01.697170865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:19:01.700846972Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:19:03.840511433Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T18:19:03.840537238Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T18:19:04.600706976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:19:04.60432052Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:19:06.554500715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:06.732515454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:19:06.736076758Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:19:07.368306589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:19:07.371792588Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:19:07.697775555Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=debug ts=2026-02-16T18:19:08.722579974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:19:11.554529284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:11.554607412Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
ts=2026-02-16T18:19:13.290327265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:19:14.651230976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:19:14.656233092Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:19:15.412203075Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-02-16T18:19:16.55378336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:19.021255182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:19:19.02526321Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:19:19.268811985Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:19:21.554029326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:22.804177114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:19:22.810354779Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:19:23.126336652Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:19:23.335119108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:19:23.34009558Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:19:26.554032491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:19:26.554063302Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:19:26.554071907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:19:26.554089347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:26.98334118Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-02-16T18:19:27.394307745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:19:27.399146856Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:19:27.783758063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:19:27.787947582Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:19:28.489817614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:19:28.493872416Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:19:30.089794254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:19:30.093865599Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:19:30.840922234Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-02-16T18:19:31.553693681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:31.585927946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:19:31.590078005Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:19:34.69749936Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-02-16T18:19:35.556232324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:19:35.560624932Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:19:36.554426737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:38.554618113Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T18:19:38.669939146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="503 kB" avg_uncomp="251 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:19:38.673378591Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T18:19:40.367507513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:19:41.554539817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:41.829508993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:19:41.833712242Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:19:42.282472712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:19:42.289790298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T18:19:42.412217515Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T18:19:43.2521758Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:19:43.292509948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:19:45.18691512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:19:45.19287926Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:19:46.007331232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:19:46.010767782Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:19:46.268531662Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-02-16T18:19:46.554362054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:19:47.926576871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:19:47.930144005Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:19:48.109466902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:19:48.11300615Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T18:19:50.126533996Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T18:19:50.187762897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:19:50.191202156Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:19:50.894457894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:19:50.899833246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:19:51.409294366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:19:51.412608371Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:19:51.553794794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:53.983650307Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
ts=2026-02-16T18:19:54.735607984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50572"
level=info ts=2026-02-16T18:19:55.508981956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:19:55.512598697Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:19:56.534666864Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:19:56.535791426Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:19:56.554201043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:19:56.554221285Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:19:56.554231758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:19:56.554248026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:19:57.840897127Z caller=checkpoint.go:378 msg="writing series" size="251 kB"
level=info ts=2026-02-16T18:19:59.793775139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:19:59.800554189Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:19:59.935118002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:19:59.938679414Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:20:01.554526576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:01.697252168Z caller=checkpoint.go:378 msg="writing series" size="438 kB"
level=debug ts=2026-02-16T18:20:01.697290363Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=15
level=debug ts=2026-02-16T18:20:05.554379378Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-02-16T18:20:06.554191675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:07.133864989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:20:07.138578135Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:20:08.956077687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:20:09.411717535Z caller=checkpoint.go:378 msg="writing series" size="515 kB"
level=warn ts=2026-02-16T18:20:11.554341635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:12.369531323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:20:12.373990517Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:20:13.269151817Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
ts=2026-02-16T18:20:13.294451032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=warn ts=2026-02-16T18:20:16.553779598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:17.126023187Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:20:17.568470471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:20:17.573338873Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:20:17.573363343Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:20:17.573401016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:20:17.5768829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:20:18.789495472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:20:18.793177044Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:20:20.208201677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:20:20.212060237Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:20:20.212083614Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:20:20.21210889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:20:20.215706588Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:20:20.984178574Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB"
level=debug ts=2026-02-16T18:20:20.984202794Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=5
level=warn ts=2026-02-16T18:20:21.553813643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:22.423076135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:20:22.426823262Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:20:22.47290944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:20:22.476858439Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:20:23.228468245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48328"
level=debug ts=2026-02-16T18:20:24.840469626Z caller=checkpoint.go:378 msg="writing series" size="365 kB"
level=info ts=2026-02-16T18:20:25.155371686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:20:25.161070769Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:20:26.094607413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:20:26.098447601Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:20:26.345354247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="503 kB" avg_uncomp="251 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T18:20:26.349041461Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T18:20:26.553743282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:20:26.553777264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:20:26.553785721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:20:26.553824042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:28.697095131Z caller=checkpoint.go:378 msg="writing series" size="508 B"
level=info ts=2026-02-16T18:20:29.476457449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:20:29.48027344Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:20:29.576700901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:20:29.580283611Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T18:20:30.978220242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47258"
level=info ts=2026-02-16T18:20:31.02489124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:20:31.02867523Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:20:31.554660828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:32.393287257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:20:32.396909625Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:20:32.555661387Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=info ts=2026-02-16T18:20:33.182920737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:20:33.191072795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:20:34.007532517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:20:34.01260998Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:20:35.038358809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:20:35.041746234Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:20:36.411347428Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=debug ts=2026-02-16T18:20:36.552081194Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:20:36.554129762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:36.806525184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:20:36.810178147Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:20:37.320810461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:20:37.324771068Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:20:40.269145557Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-02-16T18:20:40.589452191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:20:40.591017855Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:20:41.554641879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:41.638665262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:20:41.642107372Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T18:20:43.003762164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:20:43.00741355Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:20:43.296528208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:20:44.031244616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:20:44.034887145Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:20:44.034909488Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:20:44.034930538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:20:44.037949608Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:20:44.1261855Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-02-16T18:20:46.554192854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:47.983754016Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-02-16T18:20:51.554526132Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:20:51.839930321Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:20:54.556527515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:20:54.563349646Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:20:55.182983394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:20:55.190591343Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:20:55.440316805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:20:55.444307968Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:20:55.697116788Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T18:20:56.534734143Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:20:56.535826731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:20:56.554162861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:20:56.554185642Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:20:56.554194016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:20:56.554183122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:20:58.348369167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:20:58.352963029Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:20:59.55508883Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:21:00.132832916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:21:00.136948282Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:21:01.5539047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:01.795307129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:21:01.799127141Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:21:03.411820334Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=warn ts=2026-02-16T18:21:06.554526908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:07.024404127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:21:07.028266523Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:21:07.269386481Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:21:08.241023315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:08.244799212Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:08.244818889Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:08.244842713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:08.247809389Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:21:08.455940208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:21:08.459829557Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:21:09.144990519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:21:11.126292216Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=warn ts=2026-02-16T18:21:11.554304089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:12.17239949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:12.176184918Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:12.176205879Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:12.176229114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:12.179268001Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:21:13.209287651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:21:13.217233456Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:21:13.217261385Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:13.217291763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:21:13.224374406Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-02-16T18:21:13.299780318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:21:14.366548362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:14.370413421Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:14.370437427Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:14.370460254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:14.373816799Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:21:14.982815067Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T18:21:14.985912682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:21:14.990964153Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:21:15.948073404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="503 kB" avg_uncomp="251 kB" full=2 labels="{stream=\"image-controller\"}"
level=info ts=2026-02-16T18:21:15.949144778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:21:15.951410855Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c679e9f14%3A19c679e9f15%3Acd409df9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T18:21:15.951429186Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:15.951449053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="50 kB" avg_uncomp="50 kB" idle=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:21:15.953581462Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:21:15.953589735Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:21:16.127075995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:16.130392257Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:16.13041313Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:16.251698883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:16.255171761Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:21:16.554548113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:18.840803581Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:21:20.849271034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:20.854208081Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:20.854230427Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:20.854251662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:20.85990288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:21:21.553680877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:22.291717049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:21:22.69814632Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:21:22.761485819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.3 kB" total_uncomp="55 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:21:22.764833418Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c679dc0d7%3A19c679e9384%3A6c42857e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T18:21:22.764856081Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:22.764878218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="395 kB" avg_comp="395 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" idle=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:21:22.773031485Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:21:22.863276305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:21:22.866645502Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:21:25.710837324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:21:25.716005458Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:21:26.553498911Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:21:26.553535935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:21:26.55354449Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:21:26.553587152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:21:26.55462336Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=16
level=info ts=2026-02-16T18:21:26.560730845Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-02-16T18:21:26.56346628Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00936506s
level=info ts=2026-02-16T18:21:27.126423157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:27.129702953Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:21:27.688392762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:21:27.691817887Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:21:28.782364945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:21:28.78576125Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:21:29.721539498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:21:29.728337388Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:21:30.644427281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:21:30.64803589Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:21:30.67810788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:21:30.681429393Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:21:31.554256963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:33.183930788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:21:33.187579072Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:21:34.382220489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:21:34.512466615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:21:34.516365007Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:21:34.570706213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:21:34.574139594Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:21:36.55423782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:38.911791652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:21:38.916146745Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:21:39.482934796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:39.486555756Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:39.486575003Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:39.486598443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:39.490121918Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:21:39.61672224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:21:39.623331998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:21:40.427415616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:21:40.432872022Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:21:41.553838074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:42.253158052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:42.256994985Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:42.25701709Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:21:42.257038449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:42.260116522Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:21:43.30159545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:21:44.217973168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:21:44.222147747Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:21:46.553960215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:49.47485635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:21:49.478657258Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:21:49.555162864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:21:49.559037869Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:21:51.554257882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:53.562272949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:21:53.570372908Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:21:53.767938483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:21:53.772239714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T18:21:54.7425243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50216"
level=info ts=2026-02-16T18:21:55.301351204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:21:55.305291374Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:21:56.5343429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:21:56.535390217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:21:56.553766823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:21:56.553770036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:21:56.553763921Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-02-16T18:21:56.553788379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:21:56.553796699Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:21:56.553928512Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-02-16T18:21:56.554230134Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=info ts=2026-02-16T18:21:56.738912709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:21:56.743079239Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:21:57.638251116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:21:57.642023267Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:21:58.376151861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:21:58.379497564Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:21:59.044589987Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:22:00.209128385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:00.213189778Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:00.213213272Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:00.213235073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:00.216573606Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:22:00.360159222Z caller=checkpoint.go:378 msg="writing series" size="5.3 MB"
level=debug ts=2026-02-16T18:22:00.360182967Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.5 MB" series=2
level=info ts=2026-02-16T18:22:00.395270002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:22:00.39914984Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:22:00.991055855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37876"
level=warn ts=2026-02-16T18:22:01.553751927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:02.0746844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:02.078051421Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:02.078073101Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:02.078093328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:02.081046406Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:22:03.168472012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:22:03.172048456Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:22:04.16054158Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:22:06.554559555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:07.182733868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:22:07.186663858Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:22:07.963215449Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T18:22:08.946973959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:22:09.235531766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:22:09.240949659Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:22:10.446593674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:22:10.450037764Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:22:11.169288211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:11.172615108Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:11.172637964Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:11.172660043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:11.175494539Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:22:11.554392403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:11.765938202Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-02-16T18:22:12.016116177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:22:12.019928025Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:22:13.153101547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:13.156549551Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:22:13.30300036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:22:15.214971275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:15.218262676Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:22:15.569133565Z caller=checkpoint.go:378 msg="writing series" size="716 kB"
level=warn ts=2026-02-16T18:22:16.553709325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:19.371279053Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-02-16T18:22:19.37130416Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-02-16T18:22:19.769539565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:22:19.773179018Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:22:20.473742553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:22:20.47730563Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:22:21.376900439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:22:21.380759634Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:22:21.554090781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:21.778705771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:22:21.782622795Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:22:21.782650663Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:21.782675772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:22:21.785968303Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:22:23.174031533Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
ts=2026-02-16T18:22:24.743440184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39076"
level=info ts=2026-02-16T18:22:25.983778745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:22:25.987928992Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:22:26.425234318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:22:26.432615156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:22:26.554830294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:22:26.554867504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:22:26.554878713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:22:26.554911635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:26.977127881Z caller=checkpoint.go:378 msg="writing series" size="5.6 kB"
level=info ts=2026-02-16T18:22:27.512280208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:22:27.517084031Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:22:27.517106126Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:22:27.51713142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:22:27.520281876Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:22:30.780901875Z caller=checkpoint.go:378 msg="writing series" size="724 kB"
level=info ts=2026-02-16T18:22:31.039296165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:31.043395719Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:22:31.554238466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:31.869783292Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:22:33.663603292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:22:33.668853093Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:22:34.583390966Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=debug ts=2026-02-16T18:22:34.583414585Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-02-16T18:22:35.048743003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:22:35.052289665Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:22:36.553588072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:37.892743913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:22:37.896577355Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:22:38.386237156Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T18:22:41.22316363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:22:41.227408816Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:22:41.554391531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:41.922764768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:22:41.926041525Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:22:41.997052562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:22:42.000581279Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:22:42.188966355Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
ts=2026-02-16T18:22:43.305425406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:22:44.900140146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:22:44.904537265Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:22:45.531836457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:22:45.991215045Z caller=checkpoint.go:378 msg="writing series" size="563 kB"
level=warn ts=2026-02-16T18:22:46.553901736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:47.98650204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:22:47.993504769Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:22:48.225125901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:22:48.228817061Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:22:48.301902896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:22:48.305915904Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:22:49.793842816Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:22:50.977414239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:22:50.981047167Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:22:51.475370664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:22:51.478820484Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:22:51.553754846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:22:53.59701064Z caller=checkpoint.go:378 msg="writing series" size="542 kB"
level=debug ts=2026-02-16T18:22:53.597040021Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-02-16T18:22:53.78529202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:22:53.788824899Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:22:55.427691456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:22:55.431359982Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:22:56.534811831Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:22:56.535896222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:22:56.55426141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:22:56.554269685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:22:56.554289911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:22:56.554297691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:22:57.177363706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:22:57.181421221Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:22:57.399744493Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=debug ts=2026-02-16T18:22:57.94517268Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:22:58.738387385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:22:58.741860631Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:23:01.203151154Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T18:23:01.203181217Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-02-16T18:23:01.55402415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:02.224810968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:23:02.228470065Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:23:05.00524822Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=info ts=2026-02-16T18:23:06.01395413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:23:06.017834411Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:23:06.55374647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:07.973108085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:23:07.979023653Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:23:08.808728524Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-02-16T18:23:11.554512828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:23:12.611226202Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:23:13.231032898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:23:13.236062278Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:23:13.307527148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:23:15.707939132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:23:15.715221915Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:23:16.413404196Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T18:23:16.553609807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:16.77626922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:23:16.780569389Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:23:18.233212786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:23:18.2370933Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:23:18.818357452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:23:18.822351764Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:23:18.909600287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:23:18.917718508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:23:19.841467065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:23:19.845176564Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:23:20.216626577Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:23:21.554151049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:21.820324484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:23:21.824139012Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:23:23.047518106Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:23:24.019427187Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
ts=2026-02-16T18:23:24.74589148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52100"
level=info ts=2026-02-16T18:23:26.554238142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:23:26.554267721Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:23:26.554275824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:23:26.554281798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:26.631885607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:23:26.635716758Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:23:27.822280898Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:23:29.597759855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:23:29.602423628Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:23:29.602459742Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:23:29.602516228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:23:29.606165542Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:23:30.017814696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:23:30.025218803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T18:23:31.554114024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:23:31.625249935Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T18:23:33.327749066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:23:33.331060853Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:23:35.298616967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:23:35.30330745Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:23:35.303347873Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:23:35.303375822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:23:35.307387569Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:23:35.428032214Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T18:23:36.553627973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:23:37.822658205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:23:38.895943005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:23:38.900715703Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:23:39.247936658Z caller=checkpoint.go:378 msg="writing series" size="17 MB"
level=debug ts=2026-02-16T18:23:39.247961146Z caller=checkpoint.go:390 msg="flushing series" totalSize="17 MB" series=10
level=info ts=2026-02-16T18:23:40.690468778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:23:40.694376908Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T18:23:41.554634827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:23:43.03337464Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-02-16T18:23:43.238026928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" idle=1 full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:23:43.24178039Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:23:43.309090515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:23:45.399978756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:23:45.404250541Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:23:45.694091959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:23:45.698366753Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:23:46.553937003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:46.711364636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:23:46.715290338Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:23:46.836555425Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=info ts=2026-02-16T18:23:47.306016165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:23:47.310163838Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:23:47.564160107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:23:47.568387891Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:23:49.044296865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:23:49.048452276Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:23:50.021315557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:23:50.02491072Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:23:50.639276744Z caller=checkpoint.go:378 msg="writing series" size="824 kB"
level=debug ts=2026-02-16T18:23:50.639304937Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-02-16T18:23:51.554577778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:23:53.244953508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33948"
level=debug ts=2026-02-16T18:23:54.442163274Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
ts=2026-02-16T18:23:54.747696778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58078"
level=debug ts=2026-02-16T18:23:55.760112828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:23:56.534776343Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:23:56.535858084Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:23:56.554294445Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:23:56.554317489Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:23:56.554325676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:23:56.554349672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:23:56.883010324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:23:56.886508822Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:23:58.245252996Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=info ts=2026-02-16T18:24:00.044757004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:24:00.048286477Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:24:01.554113612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:02.048004219Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-02-16T18:24:05.850741774Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=info ts=2026-02-16T18:24:05.85781288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:24:05.861515614Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:24:06.554456134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:07.109837218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:24:07.115527797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:24:08.873033835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:24:08.876501079Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:24:09.652954409Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:24:10.502314814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="439 B" avg_comp="439 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:24:10.506937492Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:24:11.13009673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:24:11.134363669Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:24:11.554504816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:12.631164141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:12.638037162Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:12.638060179Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:24:12.63808588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:12.644571074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:24:13.310666833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:24:13.456069841Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=info ts=2026-02-16T18:24:13.568103424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:24:13.572356989Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:24:13.817448115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:24:13.821709091Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:24:14.861793671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:24:14.865465969Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:24:15.597690543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:24:15.60169734Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:24:15.601733422Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:24:15.601766157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:24:15.6060473Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:24:16.554363599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:16.809829032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:24:16.815278621Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:24:17.259522234Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-02-16T18:24:20.180311091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:24:20.183867713Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:24:21.061927419Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:24:21.554289298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:21.808560451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:24:21.810506778Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:24:22.434268242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:24:22.437466102Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T18:24:23.246855348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43720"
level=info ts=2026-02-16T18:24:24.655642015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:24:24.664238193Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T18:24:24.864687117Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:24:26.554445316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:24:26.55449442Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:24:26.55450599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:24:26.554495183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:24:27.607265548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:24:28.66739263Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=info ts=2026-02-16T18:24:30.135897787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:24:30.139351514Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:24:31.00962858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45912"
level=warn ts=2026-02-16T18:24:31.554552971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:32.124584978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:24:32.134107877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:24:32.470083533Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:24:32.470105654Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T18:24:33.567238934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:24:33.571585976Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:24:36.046143618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:24:36.049821147Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:24:36.273438286Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-02-16T18:24:36.554567558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:37.109300134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:24:37.11297209Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:24:37.957828306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:24:37.961437285Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:24:40.075817207Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-02-16T18:24:41.554086845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:42.761905504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:24:42.765462281Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:24:43.313009588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:24:43.878413212Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:24:45.323278636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:24:45.327595234Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:24:45.601003789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:24:45.604999814Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:24:45.897832956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:24:45.9019122Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:24:46.152743567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:24:46.156155137Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:24:46.553677397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:47.280979888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:24:47.286072838Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:24:47.681511285Z caller=checkpoint.go:378 msg="writing series" size="119 kB"
level=info ts=2026-02-16T18:24:47.838083793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:24:47.84156565Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:24:51.484334632Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=warn ts=2026-02-16T18:24:51.554399333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:24:53.248040131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53884"
level=info ts=2026-02-16T18:24:55.205851849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:24:55.209936139Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:24:55.287311143Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-02-16T18:24:56.53439878Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:24:56.535494798Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:24:56.553827688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:24:56.553853042Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:24:56.553831767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:24:56.553863922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:24:59.090257025Z caller=checkpoint.go:378 msg="writing series" size="333 kB"
level=debug ts=2026-02-16T18:25:00.115193253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:25:00.496548803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:25:00.749528105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="457 B" avg_comp="457 B" total_uncomp="340 B" avg_uncomp="340 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:25:00.753135303Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:25:01.553710909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:02.893103331Z caller=checkpoint.go:378 msg="writing series" size="452 kB"
level=info ts=2026-02-16T18:25:04.336070296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:25:04.339518927Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:25:05.784595224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:25:05.788221819Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:25:05.788242632Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:25:05.788265886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:25:05.790984602Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:25:05.849252602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:25:05.852535082Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:25:06.497514518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:25:06.501028548Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:25:06.553956904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:06.695389844Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=debug ts=2026-02-16T18:25:06.695414824Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-02-16T18:25:08.595054106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:25:08.603094697Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:25:10.100340102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:25:10.103968746Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:25:10.498101024Z caller=checkpoint.go:378 msg="writing series" size="488 kB"
level=warn ts=2026-02-16T18:25:11.55455728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:12.965283042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:25:12.968608304Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T18:25:13.315778122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:25:13.459924149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:25:13.463241809Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:25:14.301579867Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=warn ts=2026-02-16T18:25:16.553938188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:16.705333703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:25:16.715059439Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:25:17.984120056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:25:17.989589342Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:25:18.103796073Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:25:20.490573893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:25:20.4960976Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:25:20.907534583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:25:20.911356352Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:25:21.554026935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:21.911637316Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T18:25:21.911666947Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=4
level=info ts=2026-02-16T18:25:22.259454788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:25:22.263896971Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:25:23.598689653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:25:23.602402563Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:25:24.755131846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42978"
level=debug ts=2026-02-16T18:25:25.709378881Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T18:25:26.251444083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:25:26.254922653Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:25:26.553574424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:25:26.553615809Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:25:26.553624552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:25:26.553662912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:29.512438878Z caller=checkpoint.go:378 msg="writing series" size="508 B"
level=info ts=2026-02-16T18:25:30.033306412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:25:30.039204336Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:25:30.632626882Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:25:31.015813626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48400"
level=warn ts=2026-02-16T18:25:31.553843926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:33.315244517Z caller=checkpoint.go:378 msg="writing series" size="295 kB"
level=info ts=2026-02-16T18:25:33.998402276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:25:34.001951506Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:25:34.111799449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:25:34.115467904Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:25:34.738695522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:25:34.742305831Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:25:34.779393007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:25:34.783056399Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:25:36.554358318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:36.957541132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:25:36.961183315Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T18:25:37.118444991Z caller=checkpoint.go:378 msg="writing series" size="5.6 kB"
level=info ts=2026-02-16T18:25:39.550456517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:25:39.5541126Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:25:40.920834357Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T18:25:41.55460767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:41.683413629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" idle=1 full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:25:41.687261312Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T18:25:43.318639715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:25:43.472697507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:25:43.477949925Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:25:44.723626873Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-02-16T18:25:46.554339037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:46.792315006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:25:47.621544613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:25:47.625067951Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:25:48.526565499Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:25:50.43294086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:25:50.442579047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:25:50.466596814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:25:50.469816129Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:25:50.687413094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:25:50.690939533Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:25:51.554314996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:25:52.329505627Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
ts=2026-02-16T18:25:53.252441306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39778"
ts=2026-02-16T18:25:54.75630387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46210"
level=debug ts=2026-02-16T18:25:56.131823114Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-02-16T18:25:56.534551679Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:25:56.535619204Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:25:56.553963978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:25:56.553986189Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:25:56.553995067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:25:56.55400007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:25:58.734578079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:25:58.738113398Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:25:59.027948477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:25:59.031373952Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:25:59.935074304Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:26:00.779816545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="384 kB" avg_uncomp="384 kB" idle=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:26:00.779863688Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T18:26:00.784779728Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:26:01.30415513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:26:01.308456977Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:26:01.414815774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:26:01.419267782Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:26:01.554539014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:26:03.73773786Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-02-16T18:26:06.554136233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:06.654544422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:26:06.658445068Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:26:07.132259945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:26:07.135825546Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:26:07.540575184Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:26:07.867400572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:26:07.872143658Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:26:11.343848716Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=warn ts=2026-02-16T18:26:11.553621692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:12.244454451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:26:12.248679233Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
ts=2026-02-16T18:26:13.320607671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:26:13.734786392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:26:13.740670481Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:26:15.146622166Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T18:26:15.147679687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="140 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:26:15.155325972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T18:26:16.554364314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:26:18.949214936Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:26:20.310940028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:26:20.315728881Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:26:21.553975963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:21.585125425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:26:21.588850599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:26:22.271787847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:26:22.275129245Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:26:22.751901286Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:26:22.751924378Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=16
level=info ts=2026-02-16T18:26:25.564453114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:26:25.568726985Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:26:26.553865231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:26:26.553896633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:26:26.553906287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:26:26.553912215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:26.561336927Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-02-16T18:26:26.564426979Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010312088s
level=info ts=2026-02-16T18:26:29.870053902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:26:29.873855452Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:26:30.114145395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:26:30.117916963Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T18:26:31.021990954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47680"
level=warn ts=2026-02-16T18:26:31.554030358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:32.113356563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:26:32.116977434Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:26:33.416262925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:26:33.891633644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:26:33.895596147Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T18:26:35.265784596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:26:35.269333878Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:26:35.91996565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:26:35.923976788Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:26:36.126597125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:26:36.12976206Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:26:36.470798536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:26:36.474376143Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:26:36.553597952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T18:26:41.554373514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:41.730053784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:26:41.733618191Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:26:41.733637513Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:26:41.733659266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:26:41.736863039Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:26:43.32214978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:26:45.305754116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:26:45.792063055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:26:45.795568443Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:26:46.553693836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:46.617865485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:26:46.626639722Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:26:47.298037119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:26:47.301883866Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:26:48.257788229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:26:48.261184494Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:26:49.697982498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:26:49.701663594Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:26:51.553643054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:26:53.256720705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60482"
level=info ts=2026-02-16T18:26:53.475801083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:26:53.481203887Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:26:54.759047667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47176"
level=info ts=2026-02-16T18:26:55.219618613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:26:55.223099626Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:26:55.223119836Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:26:55.223142207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:26:55.225882546Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:26:56.268995597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:26:56.272500699Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:26:56.534640572Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:26:56.535702611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:26:56.55411308Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:26:56.554116411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:26:56.554142874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:26:56.554142769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:26:56.554152042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:26:56.554233996Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-02-16T18:26:56.554549119Z caller=checkpoint.go:378 msg="writing series" size="347 kB"
level=info ts=2026-02-16T18:26:56.710895585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:26:56.714643433Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:26:57.730277724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:26:57.735232223Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:27:00.361057429Z caller=checkpoint.go:378 msg="writing series" size="5.6 MB"
level=debug ts=2026-02-16T18:27:00.361082981Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.0 MB" series=2
level=info ts=2026-02-16T18:27:00.563771933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="140 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:27:00.570710719Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T18:27:00.71300113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:27:00.716894029Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:27:01.025635561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52984"
level=warn ts=2026-02-16T18:27:01.554127135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:03.141929857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:27:03.14559785Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:27:04.160816995Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:27:06.554222758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:06.587396798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:27:06.591053775Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:27:07.957281788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:27:07.96333604Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-02-16T18:27:10.95875166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:27:10.962245578Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:27:11.553767822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:11.766730993Z caller=checkpoint.go:378 msg="writing series" size="170 kB"
ts=2026-02-16T18:27:13.323702124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:27:15.137907767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:27:15.142881182Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:27:15.178951911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:27:15.182677463Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:27:15.56930388Z caller=checkpoint.go:378 msg="writing series" size="694 kB"
level=info ts=2026-02-16T18:27:16.032293507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:27:16.036034187Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:27:16.553849747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:18.945530031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:27:18.949194565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:27:19.371470092Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=debug ts=2026-02-16T18:27:19.371508221Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-02-16T18:27:21.554306527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:22.40573601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:27:22.409113338Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:27:22.409134Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:22.409155918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:27:22.412021405Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:27:23.174506249Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T18:27:23.198607153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:27:23.201772948Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T18:27:23.258213095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57040"
level=debug ts=2026-02-16T18:27:23.792658299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:27:24.620028803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:27:24.623375159Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:27:26.156440664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:27:26.160226727Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:27:26.553858635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:27:26.55390368Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:27:26.553912226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:27:26.553919037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:26.977791611Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-02-16T18:27:28.240601088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:27:28.244194466Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:27:30.780684746Z caller=checkpoint.go:378 msg="writing series" size="837 kB"
level=warn ts=2026-02-16T18:27:31.554195067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:32.237969903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:27:32.241613468Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:27:34.583343472Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:27:35.067800417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:27:35.072076958Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:27:35.482498168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:27:36.473519327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:27:36.477204907Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:27:36.554155642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:38.386493307Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T18:27:38.386531863Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-02-16T18:27:40.263166538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:27:40.267574378Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:27:40.267595602Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:40.267617116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:27:40.270902347Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:27:40.894806586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:27:40.898240042Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:27:40.983531031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:27:40.989728404Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:27:41.393135039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:27:41.397324983Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:27:41.397352932Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:41.397386587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:27:41.40131959Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:27:41.554291931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:27:42.188569153Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
ts=2026-02-16T18:27:43.325169286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:27:43.403227982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:27:43.407176258Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:27:44.102396797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:27:44.106847261Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:27:44.644611406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:27:44.64820137Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:27:44.732518743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:27:44.735951375Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:27:45.991580695Z caller=checkpoint.go:378 msg="writing series" size="481 kB"
level=warn ts=2026-02-16T18:27:46.554444527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:47.092148843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:27:47.097676203Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:27:48.165040153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:27:48.170744433Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:27:48.170767816Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:48.17079173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:27:48.176138771Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:27:49.794208247Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T18:27:51.554143125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:51.618578413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:27:51.624992736Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:27:51.763415404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:27:51.76733752Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:27:51.767360172Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:51.767384181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:27:51.770534309Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:27:52.713764174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:27:52.721928634Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:27:53.250767853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="140 MB" avg_uncomp="23 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:27:53.257923839Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c679e60cd%3A19c67a58d4a%3Ae0fe5d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T18:27:53.257952205Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:53.257982001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:27:53.262102635Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:27:53.598898348Z caller=checkpoint.go:378 msg="writing series" size="657 kB"
level=debug ts=2026-02-16T18:27:53.598933792Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-02-16T18:27:55.47422787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:27:55.512033318Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:27:56.534158749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:27:56.535195162Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:27:56.553631809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:27:56.553642163Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:27:56.553661453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:27:56.55366904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:27:57.402583139Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:27:59.146906292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:27:59.150468999Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:27:59.15051498Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:27:59.150538445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:27:59.153397512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:28:00.577076195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:28:00.580545495Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:28:01.010550737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:01.0192089Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:01.019259051Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:01.019315575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:01.023292158Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:28:01.032257995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53966"
level=debug ts=2026-02-16T18:28:01.207225093Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-02-16T18:28:01.207262418Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=2
level=warn ts=2026-02-16T18:28:01.553800422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:01.56492164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:28:01.569884981Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:28:04.118508754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:28:05.005265996Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=info ts=2026-02-16T18:28:05.750055407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:28:05.753695029Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:28:06.461876063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:06.465336443Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:06.4653576Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:06.465381938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:06.46870519Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:28:06.553748869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:06.96655196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:28:06.970036491Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:28:07.011561424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:28:07.019017366Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c679e60cc%3A19c679e9f14%3Ae1b06d9a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T18:28:07.019050828Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:07.019086678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:28:07.02676399Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:28:08.809685672Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=info ts=2026-02-16T18:28:08.928614542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:28:08.933454991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:28:11.55456734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:11.733930814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:28:11.737648388Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:28:12.18961778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:28:12.196311255Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:28:12.196343155Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:12.196375599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:28:12.211571529Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:28:12.610941472Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
ts=2026-02-16T18:28:13.32732626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:28:16.307553394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:28:16.320232338Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:28:16.417642203Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T18:28:16.553748946Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:16.71776506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:16.722242879Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:28:17.069702313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:28:17.073275471Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:28:17.118573249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:28:17.124054933Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:28:18.895578568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:28:18.91995498Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:28:19.520330528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:28:19.525411235Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:28:19.6338511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:28:19.635555943Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:28:20.218734804Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T18:28:21.554356455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:22.825002183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:28:22.828855638Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:28:24.019637704Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=info ts=2026-02-16T18:28:24.561749205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:28:24.565381989Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:28:25.28322695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:28:25.287059905Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:28:25.661804346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:28:25.666116365Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:28:26.553502585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:28:26.553536407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:28:26.553545759Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:28:26.553595123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:27.087072074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="565 B" avg_comp="282 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:28:27.09046589Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:28:27.822624188Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:28:30.252597168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:28:30.257405861Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:28:30.434931747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:30.438923941Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:30.438947047Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:30.438968963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:30.441993269Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:28:30.460078342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:30.463495587Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:30.463516887Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:30.739968561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:30.743973583Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:28:31.554029376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:28:31.625351632Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T18:28:33.057492974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:28:33.061225893Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:28:35.428761288Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T18:28:35.910202265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:28:35.916080095Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:28:36.554087282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:37.961461263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:28:37.964892621Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:28:37.964911787Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:28:37.964933754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:28:37.967709163Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:28:38.613705167Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:28:39.243936882Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-02-16T18:28:39.243963562Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=10
level=info ts=2026-02-16T18:28:39.783358648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:28:39.789790331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:28:40.63766923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:28:40.641220012Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:28:41.55455639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:42.630415798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:28:42.634630067Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:28:43.033852971Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
ts=2026-02-16T18:28:43.329877495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:28:43.844026763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:28:43.847622629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:28:45.152532815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:28:45.155975384Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:28:45.960490511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:28:45.964768667Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:28:46.554469635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:46.672892382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:28:46.676228522Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:28:46.836511537Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=info ts=2026-02-16T18:28:47.782226466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:28:47.788535559Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:28:50.639601996Z caller=checkpoint.go:378 msg="writing series" size="757 kB"
level=debug ts=2026-02-16T18:28:50.639633837Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-02-16T18:28:51.554457809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:28:54.442193238Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-02-16T18:28:55.740359583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:28:55.74631413Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:28:56.534726403Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:28:56.535955835Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:28:56.554452672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:28:56.554499616Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:28:56.554511065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:28:56.554528963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:28:57.37579556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:28:57.379874457Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:28:57.605555501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:28:57.609823235Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:28:58.24483823Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=info ts=2026-02-16T18:29:00.892494242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:29:00.89737592Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:29:01.198632529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:29:01.202533937Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:29:01.310765885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:29:01.315011888Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:29:01.315037319Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:01.315062308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:29:01.318563104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:29:01.55392569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:29:02.048095521Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:29:03.301816265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:29:03.30607159Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:29:04.571379707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:29:04.574842115Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:29:05.851132216Z caller=checkpoint.go:378 msg="writing series" size="204 kB"
level=info ts=2026-02-16T18:29:05.944511523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:29:05.948310507Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:29:06.554088439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:06.588082849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:29:06.591588796Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:29:07.070970909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:29:07.074629173Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:29:07.513531604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:29:07.517888554Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:29:07.985446614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:29:07.989246402Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:29:09.65399753Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=warn ts=2026-02-16T18:29:11.554304472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:29:13.331755163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:29:13.456596012Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=info ts=2026-02-16T18:29:13.9768161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:29:13.980757418Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:29:14.173508348Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:29:14.374034637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:29:14.377446796Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:29:15.176373854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:29:15.179873119Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:29:15.179895424Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:15.179918597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:29:15.182806466Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:29:16.554266065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:16.934823615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:29:16.938039522Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:29:17.259337545Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-02-16T18:29:18.37230933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:29:18.377750969Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:29:18.759930447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:29:21.061792407Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:29:21.329798217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:29:21.333395433Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:29:21.553841495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:22.793182603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:29:22.797414442Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:29:24.017366266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:29:24.025843003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:29:24.864850638Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T18:29:26.55360473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:26.553615172Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:29:26.553639497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:29:26.553648278Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:29:28.0027609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:29:28.009512636Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:29:28.667588479Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=debug ts=2026-02-16T18:29:28.667610996Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T18:29:31.189035167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:29:31.192468596Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:29:31.553995123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:29:32.470588683Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T18:29:33.691854147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:29:33.69661607Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:29:35.102283853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:29:35.106508899Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:29:36.27378559Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-02-16T18:29:36.553788922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:39.028676489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:29:39.033596188Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:29:40.075941504Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=debug ts=2026-02-16T18:29:41.490832154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:29:41.554146074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:42.318060261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:29:42.321843264Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T18:29:43.334074276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:29:43.704753681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:29:43.70920184Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:29:43.878805418Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:29:46.553719668Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:46.604831305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:29:46.609412997Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:29:47.681823075Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-02-16T18:29:49.246281095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:29:49.251937848Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:29:49.251964171Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:49.25198821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:29:49.255181233Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:29:49.822809886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:29:49.826531687Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:29:51.094347077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:29:51.099010033Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:29:51.484024425Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=warn ts=2026-02-16T18:29:51.554128631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:51.986348696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:29:51.989802939Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:29:51.98982279Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:29:51.989845216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:29:51.993028745Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:29:53.267598186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36016"
level=info ts=2026-02-16T18:29:54.264732631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:29:54.269213958Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:29:55.287119062Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-02-16T18:29:56.534376234Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:29:56.535436285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:29:56.55378573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:29:56.553815684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:29:56.553825019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:29:56.553863709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:29:57.382560213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:29:57.386685572Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:29:59.090570534Z caller=checkpoint.go:378 msg="writing series" size="422 kB"
level=info ts=2026-02-16T18:29:59.909700513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:29:59.9139386Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:30:01.112746641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:30:01.1168718Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:30:01.554248201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:01.67583829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:30:01.679866714Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:30:01.723697771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:30:02.893153746Z caller=checkpoint.go:378 msg="writing series" size="376 kB"
level=debug ts=2026-02-16T18:30:02.893179862Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-02-16T18:30:03.105613257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:30:03.109695483Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:30:05.575685951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:30:05.580777483Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:30:05.732368876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:30:05.735937398Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:30:06.554320312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:06.695809302Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=info ts=2026-02-16T18:30:07.126860194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:30:07.130154447Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:30:07.5484572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:30:07.551989113Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:30:10.498937542Z caller=checkpoint.go:378 msg="writing series" size="498 kB"
level=warn ts=2026-02-16T18:30:11.554242566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:11.67885961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:30:11.682854503Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:30:12.050159063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:30:12.053988227Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T18:30:13.335203695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:30:14.301842633Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:30:14.796300564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:30:14.800031107Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:30:15.422791325Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:30:15.830084921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:30:15.834356818Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:30:16.554399535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:16.909345832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:30:16.917187117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:30:17.789528915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:30:17.793293076Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:30:18.10426671Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:30:20.35313052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:30:20.35718626Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:30:21.554415811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:21.908276769Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-02-16T18:30:21.908301051Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=5
ts=2026-02-16T18:30:23.268764002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:49326"
ts=2026-02-16T18:30:24.77677916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47262"
level=debug ts=2026-02-16T18:30:25.710013868Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T18:30:26.554424194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:30:26.554454325Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:30:26.554463113Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:30:26.5544472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:28.209613568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:30:28.213626536Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:30:28.386151493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:30:28.390214979Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:30:29.512442747Z caller=checkpoint.go:378 msg="writing series" size="508 B"
level=info ts=2026-02-16T18:30:30.321058037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:30:30.324697775Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:30:31.048952037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33670"
level=info ts=2026-02-16T18:30:31.526518085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:30:31.530981228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:30:31.554179561Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:33.31551726Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=warn ts=2026-02-16T18:30:36.554394665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:37.118523583Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T18:30:39.320118836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:30:39.324405119Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:30:40.168046397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:30:40.17223962Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:30:40.694747861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:30:40.69972082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:30:40.921376056Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T18:30:41.554239353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:30:41.952978896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:30:41.958084754Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:30:42.035684203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:30:42.039509737Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:30:43.337862216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:30:43.983156307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:30:43.986840062Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:30:44.724009338Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-02-16T18:30:46.554404226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:46.950870087Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:30:47.025993309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:30:47.029605939Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:30:48.527094687Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:30:49.463288196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:30:49.468434698Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:30:50.184827406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:30:50.188275282Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:30:50.944024973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:30:50.950636499Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:30:51.553971456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:52.330036518Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:30:54.422704574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:30:54.426277447Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:30:54.778650602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35968"
level=info ts=2026-02-16T18:30:55.477770878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:30:55.481378148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:30:56.132746861Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T18:30:56.534369784Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:30:56.535440547Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:30:56.536780139Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T18:30:56.536841593Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968073 n=1
level=debug ts=2026-02-16T18:30:56.53689165Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968073 n=1
level=info ts=2026-02-16T18:30:56.553805996Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:30:56.553830199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:30:56.553839518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:30:56.553874064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:30:58.514886169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:30:58.597356308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:30:58.600641156Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:30:59.764126598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:30:59.767686272Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:30:59.830887964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:30:59.83474141Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:30:59.935142192Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:31:00.862966167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:31:00.866230269Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:31:01.554402466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:03.738566641Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=info ts=2026-02-16T18:31:03.759560134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:31:03.763568512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:31:05.684201928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:05.694227173Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:05.694254143Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:31:05.694285428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:05.700603315Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:31:06.553921671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:07.093172075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:31:07.096899284Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:31:07.540400307Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T18:31:11.343557586Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=info ts=2026-02-16T18:31:11.482814738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:31:11.486586617Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:31:11.553639601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:31:13.341197022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:31:13.668703326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:31:13.672633623Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:31:13.672656911Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:31:13.67267884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:31:13.675441725Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:31:15.146449214Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-02-16T18:31:16.553596915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:18.781316782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:31:18.784910006Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:31:18.949561143Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:31:19.158274511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:31:19.161682949Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:31:21.55413253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:21.683540299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:21.689951247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:31:22.659772535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:31:22.664831934Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:31:22.752594047Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:31:22.752618335Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=debug ts=2026-02-16T18:31:22.895664008Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:31:24.550217974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:31:24.554054511Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:31:24.915158423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:31:24.918966698Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:31:26.553848297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:31:26.553855093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:26.553879686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:31:26.553888593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:31:26.561680262Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-02-16T18:31:26.563380322Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009005555s
level=info ts=2026-02-16T18:31:26.699077272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:31:26.703342439Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:31:30.053722013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:31:30.058148436Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:31:30.419824059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:31:30.423406833Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:31:31.054696476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44394"
level=info ts=2026-02-16T18:31:31.22675184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:31:31.230334941Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:31:31.55445015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:32.072184804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:31:32.076236956Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:31:34.733037609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:31:34.736852872Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:31:34.736876673Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:31:34.736899844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:31:34.739925366Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:31:36.553632903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:39.39109127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:31:39.395935549Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:31:39.975767945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:31:39.979986301Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:31:40.167160704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:31:40.171225905Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:31:41.037500337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:31:41.043202112Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:31:41.554592607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:41.798318122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:31:41.802111896Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:31:43.343811809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:31:46.395609831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:31:46.399516599Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:31:46.554326708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:47.005224867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:31:47.008873255Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:31:48.550900595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:31:48.554429816Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:31:50.810410657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:31:50.814489924Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:31:51.553804646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:31:51.742386322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:31:51.748683834Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:31:53.055342912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:31:53.058894163Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:31:53.83454345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:31:54.783850872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57960"
level=info ts=2026-02-16T18:31:54.955957514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:31:54.959975465Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:31:55.480087292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:31:55.48382169Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:31:55.560535125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:31:55.564150667Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:31:56.534309524Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:31:56.535370791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:31:56.5537292Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:31:56.553764758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:31:56.55377413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:31:56.553783153Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:31:56.553792191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:31:56.553985548Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-02-16T18:31:56.55435892Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=info ts=2026-02-16T18:31:57.353156624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:31:57.357228628Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:31:57.55723871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:32:00.308708245Z caller=checkpoint.go:378 msg="writing series" size="6.4 MB"
level=debug ts=2026-02-16T18:32:00.308733744Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.7 MB" series=2
level=warn ts=2026-02-16T18:32:01.553744551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:02.392192295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:32:02.400767128Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:32:04.054979298Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:32:05.61830137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:32:05.622134268Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:32:05.622155805Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:32:05.622176727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:32:05.625267614Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:32:06.55359052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:07.805383336Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=info ts=2026-02-16T18:32:09.154712946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:32:09.159133737Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:32:11.554306162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:11.554405911Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-02-16T18:32:12.374396755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:32:12.379400427Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-02-16T18:32:13.345615907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:32:15.305277182Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-02-16T18:32:15.305301175Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T18:32:15.846771275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:32:15.850666554Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:32:16.553783923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:16.733257616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:32:16.737221524Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:32:19.055034378Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-02-16T18:32:20.741705191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:32:20.746138492Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:32:21.128282243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:32:21.131612626Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:32:21.554285337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:21.619399844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:32:21.622704113Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:32:22.805021585Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
ts=2026-02-16T18:32:23.273459993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33296"
ts=2026-02-16T18:32:24.784698324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50410"
level=info ts=2026-02-16T18:32:25.372856192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:32:25.376594226Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:32:26.553653185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:32:26.553683055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:32:26.553691917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:32:26.553679468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:26.554795784Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-02-16T18:32:26.749676097Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:32:27.843610175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:32:27.847534925Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:32:29.174342266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:32:29.177751456Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:32:29.811586258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:32:29.814756144Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:32:29.890980878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:32:29.894622977Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:32:30.30590366Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T18:32:30.305926693Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-02-16T18:32:30.34468156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:32:30.34823672Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:32:31.55415618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:32.735217276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:32:32.739862206Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:32:34.054734246Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:32:35.747013064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:32:35.75059788Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:32:36.553761015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:37.80463304Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T18:32:39.042675624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:32:39.0484582Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:32:39.48830127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:32:39.491650867Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:32:39.571178317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:32:39.575617521Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:32:40.35083279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:32:40.357648206Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:32:40.940506519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:32:40.944164186Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:32:41.553771297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:41.554868585Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-02-16T18:32:43.346624015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:32:44.364134633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:32:44.3683058Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:32:45.305362907Z caller=checkpoint.go:378 msg="writing series" size="851 kB"
level=debug ts=2026-02-16T18:32:45.305389557Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-02-16T18:32:46.553690361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:46.603863248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:32:46.608157096Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:32:47.354527111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:32:47.358038481Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:32:49.054572587Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=debug ts=2026-02-16T18:32:50.174454556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:32:50.176199517Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:32:51.106784541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:32:51.110522845Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:32:51.266051022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:32:51.269409644Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:32:51.554662624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:32:52.649980031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:32:52.804608979Z caller=checkpoint.go:378 msg="writing series" size="487 kB"
level=info ts=2026-02-16T18:32:53.240785071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:32:53.244948166Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-02-16T18:32:53.27508361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59084"
ts=2026-02-16T18:32:54.785647346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42966"
level=debug ts=2026-02-16T18:32:56.534085075Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:32:56.53515028Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:32:56.553589867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:32:56.553622431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:32:56.55364752Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:32:56.553658281Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:32:56.554703331Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:32:57.676797295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:32:57.681000952Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:33:00.30598671Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB"
level=debug ts=2026-02-16T18:33:00.306010135Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.9 MB" series=4
ts=2026-02-16T18:33:01.064527726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47818"
level=warn ts=2026-02-16T18:33:01.553991518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:01.57208835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:33:01.576909902Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:33:02.874966034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:33:02.882798298Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:33:04.054424382Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-02-16T18:33:06.554293642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:07.80534532Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-02-16T18:33:10.506040112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:33:10.511378638Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:33:11.407907591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="583 B" avg_comp="291 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:33:11.411532731Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:33:11.553911084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:11.554984035Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-02-16T18:33:13.347466187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:33:14.468606469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:33:14.472542216Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:33:15.171165579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:33:15.174941857Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:33:15.22401303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:33:15.227387778Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:33:15.304563446Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T18:33:16.554407002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:17.219015805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:33:17.223739191Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:33:17.22376245Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:33:17.223787214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:33:17.226716109Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:33:19.055066634Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-02-16T18:33:21.554603082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:22.130897808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:33:22.13497839Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:33:22.521042617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:33:22.524733399Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:33:22.804647231Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
ts=2026-02-16T18:33:23.276746991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55936"
level=info ts=2026-02-16T18:33:25.938668322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:33:25.943831803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:33:26.553605814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:26.553602043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:33:26.553632276Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:33:26.553640825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:33:26.55477883Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:33:26.951876001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:33:26.955257465Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:33:27.603834428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:33:27.760384113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:33:27.767217047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:33:27.877387297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:33:27.880950897Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:33:27.880972298Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:33:27.88099302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:33:27.88373385Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:33:27.988136711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:33:27.991958761Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:33:28.590743724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:33:28.594122289Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:33:30.304533789Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-02-16T18:33:31.067586365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34634"
level=info ts=2026-02-16T18:33:31.187267173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:33:31.190671186Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:33:31.554053908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:33.796528779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:33:33.805413216Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:33:34.055327906Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=warn ts=2026-02-16T18:33:36.554222579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:37.446722083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:33:37.450382616Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:33:37.826137423Z caller=checkpoint.go:378 msg="writing series" size="22 MB"
level=debug ts=2026-02-16T18:33:37.826167902Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=10
level=info ts=2026-02-16T18:33:38.209512468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:33:38.21318358Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:33:39.294355466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:33:39.297688044Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:33:39.546727819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:33:39.550423378Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:33:40.302133654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:33:40.305856045Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:33:41.554514618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:41.554634426Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=info ts=2026-02-16T18:33:43.287365154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:33:43.291030369Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:33:43.291049918Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:33:43.291070251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:33:43.294011586Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:33:43.34876276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:33:43.706908521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:33:43.710240092Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:33:44.263874911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:33:44.268122991Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:33:44.685371842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:33:44.688938729Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:33:44.930151288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:33:45.305130002Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-02-16T18:33:46.55359724Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:33:49.055188417Z caller=checkpoint.go:378 msg="writing series" size="842 kB"
level=debug ts=2026-02-16T18:33:49.055215275Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-02-16T18:33:51.554393357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:52.215920951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:33:52.220241879Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:33:52.804354468Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
ts=2026-02-16T18:33:53.278551804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41950"
level=debug ts=2026-02-16T18:33:56.534515284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:33:56.535505152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:33:56.554015942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:33:56.554017944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:33:56.554048413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:33:56.554057453Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:33:56.555372697Z caller=checkpoint.go:378 msg="writing series" size="461 kB"
level=info ts=2026-02-16T18:33:57.075275424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:33:57.079336371Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:33:57.139293622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:33:57.143436237Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:34:00.305338173Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T18:34:01.071118094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43094"
level=debug ts=2026-02-16T18:34:01.418748154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:34:01.553995498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:04.054591932Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=info ts=2026-02-16T18:34:04.53475592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:34:04.539250817Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:34:06.554540438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:07.613189659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:07.617357015Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:07.617378469Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:07.617400612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:07.620575227Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:34:07.805152085Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:34:08.056157195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:34:08.060327152Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:34:09.342147042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="886 B" avg_comp="443 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:34:09.345847832Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:34:09.488081572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:09.491953483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:09.491974953Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:09.49199873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:09.49589245Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:34:09.594491524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:34:09.597901652Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:34:10.177076041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:34:10.180451853Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:34:11.554336798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:11.554361841Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=info ts=2026-02-16T18:34:12.798019602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:34:12.805600934Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:34:13.127545359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="99 kB" avg_uncomp="99 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:34:13.131001059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:34:13.349744631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:34:15.191814863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:34:15.195219737Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:34:15.195243181Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:15.195266798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:34:15.198136708Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:34:15.304488788Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-02-16T18:34:16.554132664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:19.054978352Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:34:19.747268563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:34:19.752006144Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:34:19.752032993Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:19.752059933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:34:19.755076888Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:34:20.073828923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:20.077511989Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:34:20.253088475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:20.25656416Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:34:21.295404488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:34:21.298828098Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:34:21.303891686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:34:21.306823606Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:34:21.310907356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:34:21.315229085Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:34:21.315245339Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:21.315265465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:34:21.319637327Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:34:21.554131191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:22.804881114Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:34:23.414896937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:34:23.419893791Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:34:24.877828951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:34:24.881080602Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:34:25.163994218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:34:25.168375936Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:34:25.168401021Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:25.168424723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:34:25.1715995Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:34:26.010768216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:34:26.014379025Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:34:26.553950734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:26.553958768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:34:26.553981714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:34:26.553990492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:34:26.555268717Z caller=checkpoint.go:378 msg="writing series" size="276 kB"
level=debug ts=2026-02-16T18:34:26.555287471Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-02-16T18:34:26.731894512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:34:26.735279466Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:34:26.850695966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:34:26.854274288Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:34:29.092106699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:34:29.09584155Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:34:30.305234137Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-02-16T18:34:31.553581079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:33.391076354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:34:33.395944944Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:34:34.054525742Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T18:34:34.061627699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:34:34.066496593Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:34:35.898087278Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:34:36.435767641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:34:36.439074895Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:34:36.553570523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:37.805086983Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-02-16T18:34:40.363419252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:34:40.366996775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:34:40.36739742Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:34:40.37005378Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:34:40.473310021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:34:40.477791744Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:34:40.477815877Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:40.477845533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:34:40.481899818Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:34:41.554256261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:41.554309064Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-02-16T18:34:42.148363456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:34:42.152762608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:34:42.152787383Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:42.152811074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:34:42.156531032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-02-16T18:34:43.351096787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:34:45.304805406Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-02-16T18:34:45.974657062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:34:46.554193559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:34:48.654166585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" idle=1 full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:34:48.657610119Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:34:49.054675176Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=info ts=2026-02-16T18:34:50.521580924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:34:50.525439811Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:34:51.379135979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:34:51.383426621Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:34:51.554254066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:52.804790812Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T18:34:53.124732586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:34:53.128201796Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:34:53.281567357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37432"
level=debug ts=2026-02-16T18:34:56.534452301Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:34:56.535493829Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:34:56.553966257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:34:56.553991568Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:34:56.554001863Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:34:56.554036707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:34:56.55514078Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T18:34:57.012065745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:34:57.017253872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:34:57.411239697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:34:57.415677999Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:34:58.532154813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:34:58.536594192Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:34:58.963845229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" idle=1 full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:34:58.967402056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:34:58.967424059Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:34:58.967447082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:34:58.970363783Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:35:00.304712132Z caller=checkpoint.go:378 msg="writing series" size="350 kB"
level=debug ts=2026-02-16T18:35:00.304737374Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-02-16T18:35:00.729511024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:35:00.733303258Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:35:00.733324171Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:00.733346148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:35:00.736320416Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:35:01.554431158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:01.870145653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:01.874231316Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:35:03.225991561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:35:04.05460624Z caller=checkpoint.go:378 msg="writing series" size="501 kB"
level=info ts=2026-02-16T18:35:04.497862824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:35:04.502092252Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:35:04.700292302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:35:04.704423501Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:35:06.55418871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:07.137380073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:35:07.141749446Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:35:07.692297558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="904 B" avg_comp="452 B" total_uncomp="688 B" avg_uncomp="344 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:35:07.695972568Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:35:07.804393667Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=info ts=2026-02-16T18:35:08.76614743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:35:08.771035584Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:35:08.800082597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="33 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:35:08.806506711Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:35:09.761406102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:35:09.764827243Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:35:11.554339262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:11.55461567Z caller=checkpoint.go:378 msg="writing series" size="680 kB"
level=debug ts=2026-02-16T18:35:11.554631261Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-02-16T18:35:12.427348997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:12.430724766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:12.430745989Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:12.430768622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:12.434260663Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T18:35:13.352567513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:35:14.299294949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:35:14.302677195Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:35:14.302697793Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:14.302719587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:35:14.30545955Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:35:15.304440319Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:35:15.482986034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:35:15.486240767Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:35:16.554438691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:18.159088957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:35:18.162742374Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:35:18.691275246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:35:18.69556261Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:35:19.054773892Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:35:20.677289814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:35:20.680633239Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:35:21.554301859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:22.106289923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:35:22.109697197Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:35:22.806278476Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T18:35:22.806302848Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=3
level=debug ts=2026-02-16T18:35:22.937701208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:35:23.456896133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:35:23.46057864Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:35:23.460599601Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:23.460621076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:35:23.463630613Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:35:23.654401822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:35:23.657651608Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:35:24.798793699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54944"
level=info ts=2026-02-16T18:35:26.016540396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:26.021132068Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:35:26.55374836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:26.553783319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:35:26.553808375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:35:26.55381765Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:35:26.555063067Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T18:35:28.877985625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:35:28.881447813Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:35:30.304621835Z caller=checkpoint.go:378 msg="writing series" size="508 B"
level=warn ts=2026-02-16T18:35:31.554498381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:31.674751102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:35:31.678075926Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:35:31.899710945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:35:31.903183183Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:35:33.921324698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:35:33.929832009Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:35:34.055203518Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=debug ts=2026-02-16T18:35:34.854899865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:35:36.436350139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:35:36.440356035Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:35:36.553762746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:37.80479898Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T18:35:39.37363475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:35:39.378787892Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:35:41.553867123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:41.554907053Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
ts=2026-02-16T18:35:43.354943755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:35:45.304338572Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T18:35:46.522008747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:35:46.525897628Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:35:46.553679241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:46.597830324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:35:46.602183451Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:35:47.212773711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:35:47.216992611Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:35:48.918241748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:35:48.922083308Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:35:49.054710611Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:35:51.514795916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:35:51.518881713Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:35:51.554003785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:35:51.728813115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:35:51.733281716Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:35:51.733303621Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:35:51.733327932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:35:51.73759021Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:35:51.854633305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:35:51.858431993Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:35:52.805291655Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
ts=2026-02-16T18:35:53.286313829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52314"
level=info ts=2026-02-16T18:35:53.510351213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:35:53.514286257Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T18:35:54.80038421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36070"
level=info ts=2026-02-16T18:35:56.26867733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:35:56.274863764Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:35:56.534418734Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:35:56.535468949Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:35:56.55392699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:35:56.553949281Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:35:56.553958005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:35:56.553961791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:35:56.555069395Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:35:57.256070836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:35:57.259465572Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:36:00.304976344Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-02-16T18:36:01.266879848Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:36:01.55405508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:01.664081351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:36:01.667635467Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:36:02.266182797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:36:02.269900463Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:36:02.836697563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" idle=1 full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:36:02.840276292Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:36:02.840301771Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:36:02.840332784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:36:02.843588992Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:36:03.569740603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:36:03.57333301Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:36:03.573354746Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:36:03.573377323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:36:03.576212748Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:36:04.05439115Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=info ts=2026-02-16T18:36:04.057437812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:36:04.060927411Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:36:04.670394374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:36:04.677179209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:36:05.783667058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:36:05.787548717Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:36:06.554467379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:06.905439764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:36:06.90882401Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:36:07.804925827Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T18:36:11.554082721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:36:11.555201298Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
ts=2026-02-16T18:36:13.356862711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:36:13.933409823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:36:14.894960937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:36:14.899203808Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:36:14.988602971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:36:14.991909928Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:36:15.30506592Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T18:36:16.375413167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:36:16.378798946Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:36:16.554393731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:16.851692564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:36:16.855232224Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:36:19.055157732Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:36:20.766104334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:36:20.769416782Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:36:21.55409005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:21.664443496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:36:21.667844356Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:36:22.805301462Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:36:22.805323891Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=16
level=info ts=2026-02-16T18:36:23.604600864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:36:23.607866305Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:36:25.107362116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:36:25.111364504Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:36:25.483467032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:36:25.487948277Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:36:25.832820482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:36:25.836506658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:36:26.554334146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:36:26.554366883Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:36:26.554359057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:26.554377765Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:36:26.559789456Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-02-16T18:36:26.561799047Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007566315s
level=info ts=2026-02-16T18:36:28.614246308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:36:28.618013604Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:36:28.619127571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:36:28.746626462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:36:28.75045429Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:36:31.553970002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:36.219161914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:36:36.224128813Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:36:36.554008959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:37.943532026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:36:37.947333055Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:36:38.187926847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:36:38.192156364Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:36:41.554405383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:42.569096863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:36:42.573234412Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:36:43.013333504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:36:43.016781922Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T18:36:43.358072594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:36:44.421630285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:36:44.425992779Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:36:44.426015277Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:36:44.426039985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:36:44.428926729Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:36:46.46203429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:36:46.465736307Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:36:46.553698001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:49.295895725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:36:49.301300916Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:36:49.736924488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:36:49.740461223Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:36:51.458898102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:36:51.462346313Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:36:51.554223091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:52.415439108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:36:52.418918366Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:36:52.715755405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:36:52.717638616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:36:52.738997692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:36:52.742442021Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T18:36:54.805739598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56844"
level=debug ts=2026-02-16T18:36:56.534255429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:36:56.535301428Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:36:56.553869418Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:36:56.55392142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:36:56.553939078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:36:56.55393074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:36:56.553949897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:36:56.55417332Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-02-16T18:36:56.554530937Z caller=checkpoint.go:378 msg="writing series" size="410 kB"
level=info ts=2026-02-16T18:37:00.302371855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:37:00.307684968Z caller=checkpoint.go:378 msg="writing series" size="6.7 MB"
level=debug ts=2026-02-16T18:37:00.307705541Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=2
level=error ts=2026-02-16T18:37:00.308980039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:37:00.327328669Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:37:01.31793524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:37:01.321996974Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:37:01.554580227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:02.954587386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:37:02.957825796Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:37:04.054539635Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:37:04.544368821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:37:04.54896016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:37:06.553800379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:07.804801684Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-02-16T18:37:08.041607737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:37:08.04624573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:37:09.287728631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:37:09.291450767Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:37:09.77407373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="50 kB" avg_uncomp="50 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:37:09.777812552Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:37:09.841010461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:37:09.845365805Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:37:11.553925866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:11.555109356Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
ts=2026-02-16T18:37:13.359737655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:37:15.305588076Z caller=checkpoint.go:378 msg="writing series" size="846 kB"
level=debug ts=2026-02-16T18:37:15.305613028Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-02-16T18:37:16.554438093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:16.835921962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:37:16.839700126Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:37:17.713736496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:37:17.71719202Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:37:19.055497322Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-02-16T18:37:20.065690059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:37:20.069020112Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:37:20.504196719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:37:20.507844338Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:37:21.553902358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:21.768415224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:37:21.771765464Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:37:22.151828031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:37:22.155535661Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:37:22.655084757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:37:22.658837394Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:37:22.805371131Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=info ts=2026-02-16T18:37:22.842582673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:37:22.845840103Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:37:23.293717965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42786"
level=debug ts=2026-02-16T18:37:24.464715792Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:37:24.807066101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46226"
level=info ts=2026-02-16T18:37:26.554009409Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:37:26.554043105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:37:26.554052219Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:37:26.554053394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:26.555167212Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=debug ts=2026-02-16T18:37:30.305141623Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T18:37:30.305163768Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-02-16T18:37:31.553676375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:32.015687831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:37:32.022387518Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:37:34.055021333Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:37:34.880440167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:37:34.884224232Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:37:35.744813098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:37:35.748838609Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:37:35.806000759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:37:35.80929522Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:37:36.553630013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:36.99317629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:37:36.997859653Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:37:37.804703912Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T18:37:38.232030102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:37:38.23739393Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:37:41.554553316Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=warn ts=2026-02-16T18:37:41.554575319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:42.348898219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:37:42.352211988Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:37:43.130900513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:37:43.134335618Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:37:43.361200217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:37:43.505165973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:37:43.509834128Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:37:43.514890706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:37:43.517959379Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:37:43.947922294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:37:45.304806854Z caller=checkpoint.go:378 msg="writing series" size="686 kB"
level=warn ts=2026-02-16T18:37:46.554585707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:49.054753088Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:37:49.451017503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:37:49.455145625Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:37:49.455168385Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:37:49.455192631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:37:49.457982652Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:37:49.694893851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:37:49.907402246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="272 kB" avg_comp="136 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" idle=1 full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:37:49.910888133Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:37:51.554252363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:37:52.806707986Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=debug ts=2026-02-16T18:37:52.806733509Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-02-16T18:37:53.344260041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:37:53.351304693Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:37:53.351337465Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:37:53.351363725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:37:53.357422955Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:37:54.808080117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49602"
level=info ts=2026-02-16T18:37:54.999923796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:37:55.004261275Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:37:56.534065924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:37:56.535138882Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:37:56.553705931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:37:56.553720218Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:37:56.553739918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:37:56.553747536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:37:56.554831716Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:37:58.376385935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:37:58.380548672Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:37:58.605305832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:37:58.609068807Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:38:00.307615911Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB"
level=debug ts=2026-02-16T18:38:00.307639962Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=2
level=info ts=2026-02-16T18:38:00.822158508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:38:00.826254875Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:38:01.55372574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:02.217818963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:38:02.221681192Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:38:02.589967163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:38:02.593593671Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:38:03.81895173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:38:03.822384317Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:38:04.055254296Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-02-16T18:38:06.553796509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:07.804768439Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=info ts=2026-02-16T18:38:08.961133772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:38:08.965447559Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:38:09.502317304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:38:09.510492532Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:38:11.553987459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:11.555042173Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
ts=2026-02-16T18:38:13.364009126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:38:15.305419589Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:38:15.713803913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:38:15.717369744Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:38:15.840960941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:15.844340976Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:15.84436187Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:38:15.844384281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:15.847122028Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:38:16.138059066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:38:16.143352208Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:38:16.554395256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:17.483126522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:38:17.486509586Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:38:19.05542154Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T18:38:20.782590007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:38:20.78659194Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:38:20.968850633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:38:21.152400288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:38:21.155663499Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:38:21.554101348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:21.745166908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:38:22.804848333Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
ts=2026-02-16T18:38:23.298391211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:54474"
level=info ts=2026-02-16T18:38:26.553495936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:38:26.553527787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:38:26.553536044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:38:26.553570878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:26.554523081Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:38:27.06226287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="337 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:38:27.066026373Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:38:27.081074414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:38:27.084280686Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:38:27.501594187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:38:27.505251607Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:38:28.251360297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:38:28.254779474Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:38:29.856436925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:38:29.860302996Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:38:30.305167072Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T18:38:31.203926561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:38:31.207341797Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:38:31.554317645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:38:31.845951859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:38:31.849374472Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:38:33.131707885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:38:33.137048823Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:38:34.054893793Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-02-16T18:38:36.553683819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:37.821371531Z caller=checkpoint.go:378 msg="writing series" size="22 MB"
level=debug ts=2026-02-16T18:38:37.821396331Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=10
level=info ts=2026-02-16T18:38:38.468673774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:38:38.472254329Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:38:38.93673739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:38:38.94142845Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:38:39.410603532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:38:39.415644692Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:38:41.554200045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:41.555332758Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-02-16T18:38:43.365601427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:38:43.89771553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:38:43.905016016Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:38:44.202039043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:38:44.206071745Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:38:45.305090068Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-02-16T18:38:45.514718107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="272 kB" avg_comp="136 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:38:45.517992323Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:38:46.470099014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:38:46.473730607Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:38:46.554565572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:49.056006754Z caller=checkpoint.go:378 msg="writing series" size="968 kB"
level=debug ts=2026-02-16T18:38:49.056032055Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-02-16T18:38:51.554036261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:52.805307405Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-02-16T18:38:52.914729129Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:38:52.960075221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:38:52.964791051Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:38:53.268602914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:38:53.272408663Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:38:54.811347697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53648"
level=debug ts=2026-02-16T18:38:56.533808409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:38:56.535933022Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:38:56.554437105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:38:56.554464362Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:38:56.554491763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:38:56.554520292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:38:56.554635228Z caller=checkpoint.go:378 msg="writing series" size="345 kB"
level=debug ts=2026-02-16T18:39:00.304941953Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:39:00.331356919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:39:00.33508488Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:39:00.482358421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:39:00.485816189Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:39:01.10345539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34668"
level=warn ts=2026-02-16T18:39:01.554530834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:01.83567433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:39:04.0547236Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=warn ts=2026-02-16T18:39:06.553953695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:07.80517871Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:39:08.017820204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:39:08.021966365Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:39:08.149527323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:39:08.153091422Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:39:08.153111604Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:39:08.153133631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:39:08.155961206Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:39:10.638220609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:39:10.642423122Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:39:11.55388671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:11.555054345Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-02-16T18:39:13.366469282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:39:13.725008878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:39:13.730619483Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:39:13.951231717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:39:13.954699668Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:39:14.98452196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:39:14.987846441Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:39:15.304719113Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T18:39:16.553782359Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:18.034934606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:39:18.038378062Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:39:18.107441656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:39:18.111023408Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:39:18.137339858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:39:18.140456052Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:39:19.040978253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:39:19.045345437Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:39:19.054467594Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-02-16T18:39:19.951415414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:39:21.553592682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:22.05171524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:39:22.056262472Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:39:22.804772691Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T18:39:23.079660302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:39:23.083194013Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:39:23.301842239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34920"
level=info ts=2026-02-16T18:39:23.474955557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:39:23.479373962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:39:23.828527462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:39:23.832150684Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:39:24.584384872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:39:24.588012621Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T18:39:24.812729243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43380"
level=warn ts=2026-02-16T18:39:26.553757032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:39:26.553771072Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:39:26.553794043Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:39:26.553812635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:39:26.555161596Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=debug ts=2026-02-16T18:39:26.555174625Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-02-16T18:39:27.002291434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:39:27.005834975Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:39:27.934813378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:39:27.93844486Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:39:27.938466219Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:39:27.938520717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:39:27.941398473Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:39:30.304828493Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-02-16T18:39:31.553624588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:34.05479472Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T18:39:34.263715542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:39:34.26753425Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:39:36.554511408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:37.805283616Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T18:39:38.64591127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="272 kB" avg_comp="136 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:39:38.651108814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:39:41.087605089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:39:41.092962092Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:39:41.554360362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:41.554449741Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=info ts=2026-02-16T18:39:41.968125449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:39:41.97158732Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:39:42.561136409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:39:42.568015539Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:39:43.367400083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:39:43.573894599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:39:43.577419365Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:39:44.50104041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:39:44.50459317Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:39:44.559970369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:39:44.564081429Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:39:45.305070513Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-02-16T18:39:45.462812155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:39:46.554489219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:49.054706278Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=info ts=2026-02-16T18:39:50.324492694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:39:50.327987711Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:39:51.244419549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:39:51.554233901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:52.805162095Z caller=checkpoint.go:378 msg="writing series" size="763 B"
ts=2026-02-16T18:39:53.302804929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44130"
level=debug ts=2026-02-16T18:39:56.534769071Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:39:56.535838434Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:39:56.554402502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:39:56.554427931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:39:56.5544396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:39:56.554439112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:39:56.554515789Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T18:39:56.680850582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:39:56.68531334Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:39:57.345346899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:39:57.349020436Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:39:58.159309143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:39:58.164582732Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:39:59.186472274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:39:59.190540276Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:40:00.305218557Z caller=checkpoint.go:378 msg="writing series" size="350 kB"
level=debug ts=2026-02-16T18:40:00.305250537Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-02-16T18:40:00.425276218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:40:00.429128468Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:40:01.553756846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:03.905214866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:40:03.908827531Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:40:04.05559054Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=warn ts=2026-02-16T18:40:06.553765899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:06.936998927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:40:06.940275562Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:40:06.940295582Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:06.940321207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:40:06.943610999Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:40:07.804700552Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
level=info ts=2026-02-16T18:40:08.070420217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:40:08.074059974Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:40:10.315176056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:40:10.319496826Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:40:11.209160823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:40:11.213686447Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:40:11.554438727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:11.554935076Z caller=checkpoint.go:378 msg="writing series" size="806 kB"
level=debug ts=2026-02-16T18:40:11.554952357Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-02-16T18:40:11.628665418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:40:11.632368004Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:40:13.369405983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:40:13.460290652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:40:13.464084646Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:40:13.918534795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:40:13.922546544Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:40:14.530783007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:40:14.53526868Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:40:15.305515435Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:40:15.581530016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:40:15.585123815Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:40:16.55439125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:16.826068663Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:40:18.113106717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="409 kB" avg_comp="409 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:40:18.116944115Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:40:19.054963458Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:40:19.096111731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:40:19.099572955Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:40:21.554533543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:22.806504304Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T18:40:22.806527455Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=3
ts=2026-02-16T18:40:23.303680547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33104"
level=info ts=2026-02-16T18:40:24.487838238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:40:24.491329674Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:40:24.491357613Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:24.792437219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:40:24.795706987Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:40:26.554291656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:40:26.55430602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:26.554321103Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:40:26.554329112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:40:26.554627811Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T18:40:29.281991776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:40:29.289555047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:40:30.305247517Z caller=checkpoint.go:378 msg="writing series" size="508 B"
ts=2026-02-16T18:40:31.11340454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34426"
level=warn ts=2026-02-16T18:40:31.553964619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:34.023594616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:40:34.027613473Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:40:34.027634697Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:34.027657706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:40:34.030728307Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:40:34.054925778Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=info ts=2026-02-16T18:40:35.242118764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:40:35.245878292Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:40:35.687156461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:40:35.690603007Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:40:36.554342031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:40:36.971437862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:40:36.976320369Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:40:37.804832051Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T18:40:37.84710256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:40:37.850763742Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:40:38.977631389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:40:38.98154229Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:40:38.981565213Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:38.981589659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:40:38.984464401Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:40:41.553747661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:41.554845885Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T18:40:41.717179412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:40:41.722176314Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:40:43.243022983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:40:43.246757956Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:40:43.262934418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:40:43.370579461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:40:44.506614654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:40:44.510168392Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:40:45.304734828Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T18:40:46.072182113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:40:46.075760784Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:40:46.553752789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:47.268271153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:40:47.269808435Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:40:47.549972441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:40:47.553385905Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:40:48.807122312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:40:48.81058056Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:40:49.055421721Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:40:50.849599482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:40:50.852908354Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:40:51.498444588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:40:51.501953999Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:40:51.501975988Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:51.501996299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:40:51.504918836Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:40:51.554143635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:52.804873498Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
ts=2026-02-16T18:40:53.305328768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47462"
level=info ts=2026-02-16T18:40:55.048949065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:40:55.052591001Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:40:56.534745062Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:40:56.535792353Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:40:56.554338494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:40:56.554358831Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:40:56.554370653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:40:56.554386771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:40:56.55440427Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:40:57.240958489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:40:57.245798055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:40:57.245826517Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:57.245849174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:40:57.251918214Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:40:57.287136494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:40:57.290322491Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:40:57.290343534Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:57.545896892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:40:57.549109263Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:40:59.67726899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:40:59.681430815Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:40:59.931511546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:40:59.935205339Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:40:59.949270938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" idle=1 full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:40:59.953057404Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:40:59.953074233Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:40:59.95309682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:40:59.957926871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:41:00.305116192Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-02-16T18:41:01.116667765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60956"
level=info ts=2026-02-16T18:41:01.249358504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:41:01.253202466Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:41:01.553701238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:02.829848962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:41:02.833437044Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:41:04.055145919Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=info ts=2026-02-16T18:41:04.405294551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:41:04.409149073Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:41:06.554021915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:07.499185377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:41:07.502941685Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:41:07.805092986Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:41:08.54069287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:41:08.544924131Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:41:08.544949973Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:08.544979884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:41:08.548791052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:41:08.833664762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:08.837147886Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:41:09.720671181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:41:09.724090597Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:41:11.554620477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:11.554662011Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=info ts=2026-02-16T18:41:12.176668173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:41:12.181712886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:41:12.259858514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:41:12.265774115Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:41:13.373283393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:41:14.882356216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:41:15.304535677Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T18:41:16.045756499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:41:16.050012608Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:41:16.554373602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:16.841837921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:16.845040222Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:16.845060493Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:16.845079876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:16.847797401Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:41:19.054726402Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:41:19.730024902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:41:19.733419066Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:41:20.865981388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:41:20.86999075Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:41:20.870011966Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:20.870032875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:41:20.873534164Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:41:21.55418734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:41:22.804571887Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:41:22.804596458Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=16
level=info ts=2026-02-16T18:41:23.383695552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:41:23.3883132Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:41:24.008598222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:41:24.015545965Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:41:24.82021708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40414"
level=info ts=2026-02-16T18:41:26.512067147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:41:26.515181518Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:41:26.553644076Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:41:26.553681744Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:41:26.553689885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:41:26.553758254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:26.562905631Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-02-16T18:41:26.565825976Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.011479763s
level=debug ts=2026-02-16T18:41:27.379373854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:41:28.112807371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:28.116259795Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:41:28.781310887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:41:28.784620641Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:41:31.554283935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:34.242402378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:41:34.246966827Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:41:35.631196965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:41:35.634607172Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:41:35.64060115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:41:35.643557118Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:41:36.554231349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:37.093462585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:41:37.097266978Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:41:39.540788145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:41:39.544541229Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:41:41.554587449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:42.964324073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:41:42.968834049Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:41:42.968853966Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:42.968874762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:41:42.971846064Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T18:41:43.375343402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:41:44.363911961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:41:44.367566169Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:41:45.626181567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:41:45.632159169Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:41:46.477601534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:41:46.481521694Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:41:46.553624238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:46.727903033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:41:46.731360905Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:41:48.616787212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:41:48.62023024Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:41:50.161854101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:41:50.161894985Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T18:41:50.166620726Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:41:51.553609371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:53.498939649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:41:53.503452235Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:41:54.016142861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:41:54.01970713Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:41:56.533904925Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:41:56.53602906Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:41:56.553467556Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:41:56.553498453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:41:56.553519101Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:41:56.553528356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:41:56.553540083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:41:56.55387739Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-02-16T18:41:56.554179195Z caller=checkpoint.go:378 msg="writing series" size="304 kB"
level=info ts=2026-02-16T18:41:57.449087088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:41:57.454029947Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:41:57.538452554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:41:57.54199123Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:41:57.892139484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:41:57.895581523Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:41:57.895601233Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:57.895621461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:41:57.898277495Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:41:58.466077154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:41:58.469204653Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:41:58.469224741Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:58.469245501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:41:58.472060485Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:41:59.505068619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:41:59.508773105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:41:59.508794946Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:41:59.508820386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:41:59.511693013Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:41:59.873744546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:41:59.877043695Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:42:00.310750612Z caller=checkpoint.go:378 msg="writing series" size="7.4 MB"
level=debug ts=2026-02-16T18:42:00.310774206Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.7 MB" series=2
level=info ts=2026-02-16T18:42:00.436031632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:42:00.440167881Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:42:00.886519322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:42:00.889866973Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T18:42:01.12315218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59854"
level=warn ts=2026-02-16T18:42:01.554448019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:02.827362921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:42:02.831179893Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:42:03.454152612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:42:03.458747453Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:42:04.05426519Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-02-16T18:42:05.28497172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:42:06.553607915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:07.804502116Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-02-16T18:42:11.08093856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:42:11.084624003Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:42:11.342526763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:42:11.346007794Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:42:11.554215083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:11.554282681Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-02-16T18:42:12.264091437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:42:12.267435906Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:42:13.377206882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:42:13.581611469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:13.584885242Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:42:15.305393418Z caller=checkpoint.go:378 msg="writing series" size="899 kB"
level=debug ts=2026-02-16T18:42:15.30541774Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-02-16T18:42:15.697248807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:42:15.701389303Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:42:16.149827091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:42:16.153307845Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:42:16.554461178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:16.903389772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:42:16.906490374Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:42:17.748936978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:42:17.755619143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:42:19.054271178Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-02-16T18:42:20.786262112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:42:20.790836079Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:42:20.790866566Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:20.790899593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:42:20.794326961Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:42:21.55386093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:22.286828818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:42:22.290595144Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:42:22.805034181Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=debug ts=2026-02-16T18:42:22.805151284Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:42:23.309357871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34522"
level=info ts=2026-02-16T18:42:25.782508876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:42:25.787446535Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:42:26.554523048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:26.554548437Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T18:42:26.55454399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:42:26.554567665Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:42:26.554575591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:42:30.306196117Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T18:42:30.30622245Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=4
level=info ts=2026-02-16T18:42:30.864294546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:42:30.868817116Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:42:31.554596091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:31.619777756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:42:31.623340114Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:42:32.396536123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:42:32.400464302Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:42:33.043669448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:42:33.047430744Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:42:34.054669948Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:42:34.506999133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:42:34.510541124Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:42:34.931001961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:42:34.934921816Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:42:35.537725793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:42:35.541020551Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:42:36.553701474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:37.260993759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:42:37.266091333Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:42:37.363551459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:42:37.367277186Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:42:37.804714658Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T18:42:40.725527453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:42:40.730635233Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:42:41.119493475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:42:41.123107432Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:42:41.554261409Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-02-16T18:42:41.554279081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:42.420111799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:42:42.424343438Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:42:42.424368899Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:42.424400224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:42:42.427658687Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:42:42.891114423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:42:42.895340095Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T18:42:43.380202543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:42:44.824296454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:42:44.828454886Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:42:45.305557122Z caller=checkpoint.go:378 msg="writing series" size="908 kB"
level=debug ts=2026-02-16T18:42:45.305581049Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-02-16T18:42:46.554292979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:42:48.17762627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:42:48.182026103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:42:49.054489285Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:42:51.016393858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:42:51.021493861Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:42:51.021517632Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:42:51.021541714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:42:51.024453578Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:42:51.127984805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:42:51.131566044Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:42:51.554367453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:52.805060029Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=debug ts=2026-02-16T18:42:55.047086245Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:42:56.533920845Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:42:56.536055696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:42:56.553496269Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:42:56.553519306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:42:56.553527929Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:42:56.553607216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:42:56.554602152Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:42:56.985693871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:42:56.989075747Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:42:59.246020767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:42:59.249627413Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:42:59.927872552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:42:59.96589433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="343 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:42:59.969302042Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:43:00.306025802Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB"
level=debug ts=2026-02-16T18:43:00.306051948Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.9 MB" series=4
level=info ts=2026-02-16T18:43:00.621016371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:43:00.624277728Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:43:00.678365445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:43:00.685575999Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:43:01.262202606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:43:01.269525149Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:43:01.269549234Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:43:01.269575172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:43:01.272431412Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:43:01.554541789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:04.055070992Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=info ts=2026-02-16T18:43:04.2747937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:43:04.278918619Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:43:05.686547397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:43:05.690286951Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:43:06.55403565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:07.804745733Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-02-16T18:43:11.554624693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:11.55463823Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-02-16T18:43:13.382878704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:43:13.46849108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:43:13.472268521Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:43:13.519361407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:43:13.524942494Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:43:13.552226779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" idle=1 full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:43:13.555976611Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:43:14.354098282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:43:14.358051141Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:43:15.304831585Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:43:15.491667255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:43:15.495238721Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:43:15.9347378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:43:15.938919635Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:43:16.553668515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:19.055031731Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T18:43:20.456598404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:43:20.460431369Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:43:21.554284016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:22.676332455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:43:22.679782141Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:43:22.804690995Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
level=info ts=2026-02-16T18:43:22.900966292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:43:22.90435735Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:43:24.831161596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52364"
level=info ts=2026-02-16T18:43:24.846799436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:43:24.850644314Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:43:26.383137851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="439 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:43:26.386637834Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:43:26.55359406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:43:26.553613631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:26.55363092Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:43:26.553642122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:43:26.554724387Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=debug ts=2026-02-16T18:43:27.688170159Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:43:29.227911426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:43:29.231997849Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:43:30.233673786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:43:30.237273442Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:43:30.304393343Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T18:43:30.543528788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:43:30.547171778Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:43:31.301887655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:43:31.306904177Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:43:31.554509567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:43:32.673210842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:43:32.676643139Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:43:34.05496232Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-02-16T18:43:34.681296581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:43:34.688403051Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:43:35.554149267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:43:35.55767761Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:43:36.554050026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:37.822668642Z caller=checkpoint.go:378 msg="writing series" size="22 MB"
level=debug ts=2026-02-16T18:43:37.822694924Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=10
level=info ts=2026-02-16T18:43:37.83432078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:43:37.837829358Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:43:39.562261193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:43:39.566975216Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:43:40.504836181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:43:40.50876271Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:43:41.554436647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:41.554448232Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=info ts=2026-02-16T18:43:42.220955621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:43:42.224543668Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T18:43:43.38539397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:43:44.356104299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:43:44.3595685Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:43:45.304168157Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T18:43:46.553787687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:48.237358795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:43:49.055379126Z caller=checkpoint.go:378 msg="writing series" size="997 kB"
level=debug ts=2026-02-16T18:43:49.05540513Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-02-16T18:43:51.279412596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:43:51.286281215Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:43:51.553886021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:52.804568157Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
ts=2026-02-16T18:43:54.832272361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51716"
level=info ts=2026-02-16T18:43:54.928952936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:43:54.932163614Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:43:56.462787544Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:43:56.534997321Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:43:56.534994613Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:43:56.553514718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:43:56.553541067Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:43:56.553549845Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:43:56.553599389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:43:56.554840481Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=info ts=2026-02-16T18:43:57.266412945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:43:57.270077453Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:43:57.826650447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:43:57.831464214Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:44:00.304776834Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:44:01.554406239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:01.713689164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:44:01.718885035Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:44:02.096505122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:44:02.102783912Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:44:04.05477751Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=warn ts=2026-02-16T18:44:06.553742632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:07.804547901Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:44:07.961141115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="890 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:44:07.964757868Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:44:08.543742465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:44:08.54696548Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:44:10.24761967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:44:10.251706869Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:44:11.554376578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:11.554383262Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
ts=2026-02-16T18:44:13.386324799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:44:14.613079166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:44:14.617533196Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:44:15.304398964Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-02-16T18:44:16.554308575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:17.018100901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:44:17.0222731Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:44:18.640897394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:44:18.645173677Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:44:18.645200652Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:44:18.645226113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:44:18.648213844Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:44:18.86477606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:44:18.868343732Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:44:19.054965361Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:44:19.749146721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:44:19.75286424Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:44:20.533092095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:44:20.536748183Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:44:21.554533607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:21.829186447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:44:21.832908109Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:44:22.699436451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="445 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:44:22.70333954Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:44:22.804697401Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-02-16T18:44:22.804724777Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=9
level=info ts=2026-02-16T18:44:23.846262048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:44:23.851376789Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:44:24.012818245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:44:24.01602274Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:44:26.553682978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:26.553708997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:44:26.553733579Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:44:26.553741422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:44:26.554941347Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=info ts=2026-02-16T18:44:26.739774608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:44:26.745037727Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:44:27.766298755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:44:27.770007125Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:44:28.152266929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:44:28.250431465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:44:28.25553147Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:44:28.469120995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:44:28.473005158Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:44:29.405156458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:44:29.408937677Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:44:30.304492291Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T18:44:31.263335424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:44:31.266668419Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:44:31.554346522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:32.759953539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:44:32.764018354Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:44:33.214155228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:33.220689945Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:33.220720486Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:44:33.220755332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:33.227113249Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:44:34.054614098Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-02-16T18:44:36.554575978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:37.805192855Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T18:44:41.398828295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:44:41.402346369Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:44:41.554278177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:41.55429386Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
ts=2026-02-16T18:44:43.388280507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:44:45.304692661Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:44:46.553865965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:47.794275918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:44:47.7980829Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:44:48.135277896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:44:48.137221664Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:44:48.875762145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:44:48.883584664Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:44:49.054666591Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=info ts=2026-02-16T18:44:49.948324074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:44:49.951738429Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:44:49.951761233Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:44:49.951783444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:44:49.954710599Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:44:51.553832837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:44:52.513063067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:44:52.516440644Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:44:52.804525377Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T18:44:54.042337618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:44:54.04719657Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:44:56.534111151Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:44:56.535181934Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:44:56.553697654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:44:56.553721586Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:44:56.55373012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:44:56.553743782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:44:56.55486786Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T18:44:57.903171463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:44:57.90701697Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:44:59.125301591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:44:59.130889075Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:44:59.805805239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:44:59.809003167Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:45:00.304968588Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=debug ts=2026-02-16T18:45:00.304996428Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-02-16T18:45:00.422161751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:45:00.425429468Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:45:00.682321309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:45:01.554275221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:01.833177445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:01.836928008Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:45:02.146619056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:45:02.149887038Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:45:04.054816223Z caller=checkpoint.go:378 msg="writing series" size="521 kB"
level=info ts=2026-02-16T18:45:06.200064839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:45:06.205221778Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:45:06.554581755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:07.805203344Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-02-16T18:45:07.836406758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="445 B" total_uncomp="1.0 kB" avg_uncomp="340 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:45:07.840816596Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:45:08.678951203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:45:08.682717588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:45:08.967629196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:45:08.971403094Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:45:10.677289581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:45:10.681836411Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:45:10.93905841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:45:10.944004878Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:45:11.296046398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:45:11.299755866Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:45:11.553921893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:11.556205949Z caller=checkpoint.go:378 msg="writing series" size="793 kB"
level=debug ts=2026-02-16T18:45:11.556224483Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-02-16T18:45:13.390711263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:45:13.820610602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:45:13.824417398Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:45:14.109000404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:45:14.114645151Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:45:14.697837628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:45:14.701718336Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:45:14.938528405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:45:14.942617815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:45:15.304544796Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:45:15.565537949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:45:15.569161048Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:45:16.554164674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:18.570924645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:45:18.578894626Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:45:19.054294917Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T18:45:21.554559573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:21.754118878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:45:21.75785226Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:45:22.807096313Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T18:45:22.80711922Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=3
level=info ts=2026-02-16T18:45:23.086598055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:45:23.090573299Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T18:45:23.325773637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40468"
level=warn ts=2026-02-16T18:45:26.553548977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:26.553550305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:45:26.553585813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:45:26.553598228Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:45:26.554699891Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T18:45:28.415937215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:45:28.42083254Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:45:30.305065254Z caller=checkpoint.go:378 msg="writing series" size="892 B"
ts=2026-02-16T18:45:31.144263856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35450"
level=warn ts=2026-02-16T18:45:31.554470382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:45:32.084036054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:45:32.087651536Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:45:33.577619497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:45:33.691890058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:45:33.6957598Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:45:34.05523449Z caller=checkpoint.go:378 msg="writing series" size="394 kB"
level=debug ts=2026-02-16T18:45:34.475355403Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:45:34.908491542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="317 B" avg_comp="317 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:45:34.912022465Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:45:36.553886929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:37.804582302Z caller=checkpoint.go:378 msg="writing series" size="8.6 kB"
level=warn ts=2026-02-16T18:45:41.554027023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:41.554155501Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
ts=2026-02-16T18:45:43.392433671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:45:44.12306722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:45:44.12687929Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:45:45.304450948Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-02-16T18:45:46.554224167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:49.05430621Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T18:45:49.369322065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:45:49.373188269Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:45:49.47644011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:45:49.480631473Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:45:51.554682251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:52.804159312Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:45:52.854411545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:45:52.858439365Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:45:53.876455911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:45:53.881162918Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:45:54.335602383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:45:54.340508942Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:45:55.477124554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:45:55.481143035Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:45:56.534894488Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:45:56.535957122Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:45:56.536152919Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T18:45:56.536196638Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968074 n=1
level=debug ts=2026-02-16T18:45:56.536244998Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968074 n=1
level=info ts=2026-02-16T18:45:56.55341748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:45:56.553610545Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:45:56.553630941Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:45:56.553638817Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:45:56.553669966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:45:56.554558152Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=error ts=2026-02-16T18:45:56.556606862Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:45:56.563613721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:45:56.568237538Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:45:57.643459813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:45:57.647742429Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:45:58.294301476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:45:58.297892808Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:45:58.297914146Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:45:58.297935345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:45:58.300727083Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:45:58.511365284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" idle=1 full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:45:58.516140001Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:45:59.381821744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:45:59.388316917Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:46:00.304834676Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T18:46:00.86939868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:46:00.873022088Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T18:46:01.147499784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43808"
level=warn ts=2026-02-16T18:46:01.554357777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:02.538468973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:46:02.542061262Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:46:02.542081424Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:46:02.542103154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:46:02.545021845Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:46:02.607135066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:46:02.611829436Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:46:04.054363867Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=debug ts=2026-02-16T18:46:04.054395105Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T18:46:04.357549808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:46:04.361070825Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:46:05.281778367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:46:05.285250037Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:46:05.287403801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:46:06.554371508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:06.69583538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:46:06.699673416Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:46:07.276082758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:46:07.279806282Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:46:07.804570668Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T18:46:11.554414593Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=warn ts=2026-02-16T18:46:11.554424417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:11.685239262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:46:11.690845126Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:46:12.990593055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:46:12.994357874Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:46:13.394897837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:46:15.304124994Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T18:46:15.759144143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:46:15.76266849Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:46:16.554207243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:19.055648638Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:46:19.855236733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:46:19.860911511Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:46:21.254132826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:46:21.257578558Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T18:46:21.553771739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:22.804633629Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T18:46:23.546297336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:46:24.841823939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56532"
level=info ts=2026-02-16T18:46:26.55389052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:46:26.553925858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:46:26.553935606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:46:26.553961582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:46:26.555014814Z caller=checkpoint.go:390 msg="flushing series" totalSize="268 kB" series=5
level=info ts=2026-02-16T18:46:26.562368617Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-02-16T18:46:26.563087921Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00906297s
level=info ts=2026-02-16T18:46:29.091244598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:46:29.094703008Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:46:30.369910987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:46:30.374949748Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:46:30.374974332Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:46:30.374998165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:46:30.378259932Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:46:30.63530402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:46:30.639270903Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:46:30.755708627Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T18:46:31.150714347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36052"
level=warn ts=2026-02-16T18:46:31.554014335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:34.949390263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:46:34.95345775Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:46:35.462718821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:46:35.46659194Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:46:36.553861939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:38.46438362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:46:38.468295984Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:46:38.807551484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:46:38.811111376Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:46:39.105900441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:46:39.110832651Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:46:39.42982861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:46:39.433509054Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:46:41.219974368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:46:41.224508561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:46:41.554044347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:46:43.398227029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:46:44.97205849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:46:44.978243347Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:46:46.415445714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:46:46.421389288Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:46:46.554405136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:46.892120852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:46:46.895922157Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:46:47.455738324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:46:47.460224568Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:46:48.06802942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:46:48.072026402Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:46:48.753500578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:46:48.757600554Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:46:49.173829658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:46:49.177883712Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:46:49.177908541Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:46:49.177932788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:46:49.181663675Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:46:50.169369754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:46:50.174460587Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:46:51.199464499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:46:51.204261032Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:46:51.55453631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:51.937088927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:46:51.943836921Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T18:46:53.332030367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39946"
level=info ts=2026-02-16T18:46:55.170551036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:46:55.175099403Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:46:56.534559041Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:46:56.535616827Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:46:56.554134306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:46:56.554157375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:46:56.554165832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:46:56.554173719Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T18:46:56.554217671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:46:56.554285806Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-02-16T18:46:56.554662518Z caller=checkpoint.go:378 msg="writing series" size="441 kB"
level=info ts=2026-02-16T18:46:57.660061486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:46:57.663709315Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:46:57.663736618Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:46:57.66376589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:46:57.666618632Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:46:58.109811335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:46:58.11346135Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:47:00.308272555Z caller=checkpoint.go:378 msg="writing series" size="7.8 MB"
level=debug ts=2026-02-16T18:47:00.308299428Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.3 MB" series=2
level=warn ts=2026-02-16T18:47:01.553813931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:02.000095481Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:47:03.755909661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:47:03.759597067Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:47:04.054940357Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:47:06.391595363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:47:06.395705674Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:47:06.554500544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:06.770160826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:47:06.77371639Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:47:07.804567065Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-02-16T18:47:08.973267634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:08.977540207Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:08.977562278Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:08.977587453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:08.980494708Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:47:10.816948502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:47:10.820729095Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:47:10.820754603Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:11.121461422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:47:11.12510756Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:47:11.554341954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:11.555497343Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
ts=2026-02-16T18:47:13.40161159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:47:14.951533736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:47:14.955206423Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:47:15.305897018Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T18:47:15.305928336Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-02-16T18:47:15.613302916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:47:16.554431462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:17.50942602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:47:17.513206447Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:47:19.054713386Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=info ts=2026-02-16T18:47:19.911027722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:47:19.915261759Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:47:20.069266363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:20.07310209Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:47:20.225637156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:47:20.231595576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:47:20.231618651Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:20.231645067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:47:20.236083863Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:47:20.712424124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:47:20.716549484Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:47:21.554417958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:22.141531554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:47:22.146505689Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:47:22.805452463Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=info ts=2026-02-16T18:47:25.145836919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:47:25.149623546Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:47:26.554207874Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:47:26.554227581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:26.554242709Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:47:26.554256597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:47:26.555379032Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:47:26.801977814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:47:26.805392828Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:47:30.305651549Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T18:47:30.305676795Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-02-16T18:47:30.752784731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:47:30.756307935Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T18:47:31.157409323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52646"
level=info ts=2026-02-16T18:47:31.333516518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:47:31.337408756Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:47:31.553765479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:33.188110202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:47:33.192010228Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:47:33.19203062Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:33.192053719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:47:33.195081832Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:47:34.054830307Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:47:34.648853976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:47:34.654272224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:47:34.654295065Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:34.65432022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:47:34.658835202Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:47:35.209669097Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:47:36.246518812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:47:36.250221767Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:47:36.317316369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:47:36.321280883Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:47:36.321305734Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:36.321331364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:47:36.324870127Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:47:36.553698932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:37.694148197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:47:37.697721945Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:47:37.805228769Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T18:47:38.003083833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:47:38.007781851Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:47:41.553814695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:41.554889578Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
ts=2026-02-16T18:47:43.403925494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:47:43.806801227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:47:43.811931075Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:47:44.18319619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:47:44.186842572Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:47:45.305506345Z caller=checkpoint.go:378 msg="writing series" size="751 kB"
level=debug ts=2026-02-16T18:47:45.305530934Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-02-16T18:47:45.448795779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:47:45.453720287Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:47:45.781750282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:47:45.785461506Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:47:45.888695698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:47:45.892310326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:47:46.554535364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:47.484170564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="436 kB" avg_comp="218 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:47:47.488447052Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:47:48.247576226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:47:48.251868709Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:47:48.251892244Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:48.251916275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:47:48.255581276Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:47:48.394026086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:47:48.397521223Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:47:49.055516808Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:47:49.756760543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:47:49.763445552Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:47:50.444870953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:47:50.451121867Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:47:50.854376026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:47:50.857752088Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:47:51.554387703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:47:52.405340891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:47:52.408838848Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:47:52.408861038Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:47:52.408887175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:47:52.411726072Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:47:52.805134837Z caller=checkpoint.go:378 msg="writing series" size="438 kB"
ts=2026-02-16T18:47:53.336694191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37700"
ts=2026-02-16T18:47:54.850232247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49788"
level=info ts=2026-02-16T18:47:56.371139473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:47:56.376300016Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:47:56.534122045Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:47:56.535198552Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:47:56.553717498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:47:56.553746669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:47:56.553757686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:47:56.553760682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:47:56.554854461Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:47:56.849916021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:47:56.85333874Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:47:59.366147411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:47:59.370586106Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:48:00.310268636Z caller=checkpoint.go:378 msg="writing series" size="5.6 MB"
level=debug ts=2026-02-16T18:48:00.31029532Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.0 MB" series=4
level=info ts=2026-02-16T18:48:00.938935994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:48:00.942541915Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:48:01.554186613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:01.964840021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:48:01.968412313Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:48:03.198100126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:03.202308821Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:48:04.054757936Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-02-16T18:48:06.554321371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:07.805534428Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-02-16T18:48:09.660031177Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:48:10.855616575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:48:10.859580551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:48:11.553712571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:11.554817001Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-02-16T18:48:12.262287242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:48:12.619058972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:48:12.624354362Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:48:13.404756063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:48:14.757898363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:48:14.761306252Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:48:14.761326399Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:14.761349114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:48:14.764059956Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:48:15.174119271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:48:15.177509676Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:48:15.304840849Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:48:15.599143947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:48:15.604271064Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:48:16.502166398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:48:16.505529668Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:48:16.554255928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:19.055403195Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T18:48:20.92451267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.4 kB" avg_comp="360 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:48:20.928162952Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:48:21.554103522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:21.773748799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:48:21.777118043Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:48:22.086245372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="437 kB" avg_comp="219 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:48:22.090059599Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:48:22.805231669Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-02-16T18:48:23.077022162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:23.082015677Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:23.082037369Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:23.082060503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:23.084974566Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:48:23.912439437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:48:23.916002086Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:48:24.851437805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:45128"
level=info ts=2026-02-16T18:48:26.554190878Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:48:26.55422435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:48:26.554239689Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:48:26.554255921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:26.555316636Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:48:26.861120508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:48:26.865596278Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:48:29.948147967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:48:29.951456526Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:48:30.305249837Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=warn ts=2026-02-16T18:48:31.553753421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:32.987498396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:32.992706779Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:32.992728152Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:32.992751788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:32.997398476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:48:34.05566859Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-02-16T18:48:34.600100966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:48:34.603790331Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:48:35.416110625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:48:35.420749414Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:48:35.71071971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:48:35.714738126Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:48:36.553659116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:37.796027706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:48:37.824523994Z caller=checkpoint.go:378 msg="writing series" size="25 MB"
level=debug ts=2026-02-16T18:48:37.82454507Z caller=checkpoint.go:390 msg="flushing series" totalSize="26 MB" series=10
level=info ts=2026-02-16T18:48:38.013757756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:48:38.017437449Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:48:38.492801728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:48:38.496140105Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:48:38.496161864Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:38.496184029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:48:38.498836157Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:48:39.77634055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="1.8 kB" avg_comp="899 B" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:48:39.779870756Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:48:40.898247077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:48:40.901559021Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:48:40.90157943Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:41.202540863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:48:41.205905314Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:48:41.464291007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:48:41.467941235Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:48:41.554012423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:41.555237998Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-02-16T18:48:42.302202615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:48:42.308911691Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:48:42.676299958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:48:42.679970878Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:48:42.679989904Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:48:42.680010099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:48:42.682723938Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:48:43.406903769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:48:44.625810498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:48:44.629396778Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:48:44.657509053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:48:44.662443776Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:48:45.305187096Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T18:48:46.553973366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:48:49.05652885Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T18:48:49.056554873Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-02-16T18:48:50.082855441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:48:50.097703414Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:48:51.28410816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:48:51.287896354Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:48:51.554596045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:52.622528625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:48:52.626409387Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:48:52.804919121Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T18:48:56.353116954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:48:56.356530198Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:48:56.53386159Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:48:56.536006673Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:48:56.554462027Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:48:56.554463813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:48:56.554506921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:48:56.554517842Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:48:56.554670847Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=debug ts=2026-02-16T18:48:56.985745453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:48:56.98730913Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:48:57.561087317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:48:57.564786445Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:48:58.709406078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:48:58.712865045Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:49:00.305366824Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T18:49:01.55419708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:04.054952545Z caller=checkpoint.go:378 msg="writing series" size="299 kB"
level=info ts=2026-02-16T18:49:04.07808153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:49:04.083491929Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T18:49:06.554501677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:07.140346451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:49:07.143949456Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:49:07.804839993Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:49:08.490787881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:49:08.494795086Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:49:09.959453751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:49:09.963306219Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:49:10.65118127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="335 B" avg_comp="335 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:49:10.654902164Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:49:11.219391738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:49:11.224218678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:49:11.341678095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:49:11.344852398Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:49:11.344872915Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:49:11.344895738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:49:11.347603371Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:49:11.483025938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:49:11.554261964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:11.555600976Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=info ts=2026-02-16T18:49:13.262353901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:49:13.267791034Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T18:49:13.409455118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:49:13.964061263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:49:13.967442164Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:49:15.30546852Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-02-16T18:49:16.553617856Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:16.852317584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:49:16.856013692Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:49:17.462047157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:49:17.465599139Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:49:17.46562006Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:49:17.465642357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:49:17.468421041Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:49:17.747522884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:49:17.751996393Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:49:17.900295652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:49:17.903751181Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:49:19.055215612Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:49:20.980372229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:49:20.985743868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:49:21.170115563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="438 kB" avg_comp="146 kB" total_uncomp="4.4 MB" avg_uncomp="1.5 MB" idle=1 full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:49:21.174248711Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:49:21.553752297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:22.804683788Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T18:49:26.389633136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:49:26.394916891Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:49:26.554146167Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:49:26.554189543Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T18:49:26.554181215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:26.554200066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:49:26.555354029Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=debug ts=2026-02-16T18:49:26.555369142Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=10
level=info ts=2026-02-16T18:49:28.86919252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:49:28.873500871Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:49:29.955221753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:49:29.959340782Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:49:30.305529267Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=debug ts=2026-02-16T18:49:31.09977167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:49:31.554525288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:31.724897552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:49:31.728736782Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:49:33.63937219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:49:33.643020991Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:49:34.055374601Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T18:49:34.56723944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:49:34.570926737Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:49:35.537618806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:49:35.541260377Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:49:35.541280802Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:49:35.541302913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:49:35.544237018Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:49:36.183809053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:49:36.187078467Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:49:36.55419479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:37.805211268Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T18:49:40.184063896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:49:40.190703104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:49:40.393304287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:49:40.396909685Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:49:41.334131765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="936 B" avg_comp="936 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:49:41.337831149Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:49:41.554280423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:41.555408698Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T18:49:41.661055339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:49:41.66433987Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:49:43.410718687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:49:44.030714305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:49:45.305405397Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:49:45.957593326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:49:45.961630847Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:49:46.554508106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:49.055513833Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:49:51.4327595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:49:51.442433554Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:49:51.553908609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:49:52.805330993Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T18:49:54.179836211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:49:54.184005636Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:49:55.380533904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:49:55.385848869Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:49:56.534632846Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:49:56.5356632Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:49:56.554076964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:49:56.554080119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:49:56.554101122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:49:56.554110524Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:49:56.555245963Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T18:50:00.224515853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:50:00.22865911Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:50:00.251752022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:50:00.257367007Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:50:00.305564456Z caller=checkpoint.go:378 msg="writing series" size="442 kB"
level=debug ts=2026-02-16T18:50:00.305590088Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-02-16T18:50:00.544595929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:50:00.549760859Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:50:01.482235704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:50:01.487113472Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:50:01.554172583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:02.687203908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:50:02.691155401Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:50:04.05569781Z caller=checkpoint.go:378 msg="writing series" size="571 kB"
level=info ts=2026-02-16T18:50:04.667051069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" idle=1 full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:50:04.670808393Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:50:04.829528111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:50:04.83305149Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:50:06.380115908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:50:06.383676398Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T18:50:06.554362384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:07.288337301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:50:07.293132921Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:50:07.804865679Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=warn ts=2026-02-16T18:50:11.554577888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:11.555544306Z caller=checkpoint.go:378 msg="writing series" size="753 kB"
level=debug ts=2026-02-16T18:50:11.555565255Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-02-16T18:50:12.092764312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:50:12.096716128Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:50:13.006616632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:50:13.010422311Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T18:50:13.412424323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:50:13.879422176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="438 kB" avg_comp="146 kB" total_uncomp="4.4 MB" avg_uncomp="1.5 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:50:13.883909818Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:50:14.603124207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:50:14.607222087Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:50:15.305387965Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:50:15.831755783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:50:15.835469717Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:50:16.554400948Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:18.080103686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:50:18.897504232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:50:18.90164556Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T18:50:19.055242174Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T18:50:21.554270239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:22.809209807Z caller=checkpoint.go:378 msg="writing series" size="2.8 MB"
level=debug ts=2026-02-16T18:50:22.809232703Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=3
level=info ts=2026-02-16T18:50:22.870054115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:50:22.875732835Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:50:23.043355283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:50:23.047016928Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:50:23.776056948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:50:23.779769997Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:50:26.553922583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:26.553946901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:50:26.553972678Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:50:26.553982301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:50:26.555347282Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=debug ts=2026-02-16T18:50:28.676417793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:50:29.080601421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:50:29.088090489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:50:30.304638932Z caller=checkpoint.go:378 msg="writing series" size="894 B"
ts=2026-02-16T18:50:31.176065429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52052"
level=info ts=2026-02-16T18:50:31.1897735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="954 B" avg_comp="954 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:50:31.193401111Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:50:31.554404278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:33.524830314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="50 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:50:33.528418617Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:50:34.055195849Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
level=warn ts=2026-02-16T18:50:36.554437066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:36.642750583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:50:36.647713251Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:50:37.805312997Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T18:50:40.044642539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:50:40.049190314Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T18:50:41.554247529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:41.555345506Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
ts=2026-02-16T18:50:43.414525509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:50:45.304911906Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T18:50:45.376359586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="510 B" avg_comp="255 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:50:45.380511011Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:50:46.55445277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:46.861664745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:50:46.868802073Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:50:46.868825765Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:50:46.868848658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:50:46.871889609Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:50:48.972719552Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:50:49.054965407Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T18:50:50.756441867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:50:50.760023794Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:50:51.553692479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:50:51.753266358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:50:51.757203122Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:50:52.805597667Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
ts=2026-02-16T18:50:53.350701151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52934"
level=info ts=2026-02-16T18:50:54.483228645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:50:54.487876856Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:50:54.623405121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:50:54.627266599Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T18:50:54.86127505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36094"
level=info ts=2026-02-16T18:50:56.492446789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:50:56.498101437Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:50:56.534029718Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:50:56.535097867Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:50:56.553502245Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:50:56.553524607Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:50:56.553532572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:50:56.553610613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:50:56.554639027Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:50:57.027368074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:50:57.030903088Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:50:57.470699895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="438 kB" avg_comp="146 kB" total_uncomp="4.4 MB" avg_uncomp="1.5 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:50:57.474718254Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:50:59.696241051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:50:59.700785551Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:50:59.765136737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:50:59.768951211Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:51:00.239946276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:51:00.244460633Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:51:00.304638087Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-02-16T18:51:01.179951607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53618"
level=warn ts=2026-02-16T18:51:01.554468559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:02.090194158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:02.094035647Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:51:04.055163596Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=info ts=2026-02-16T18:51:04.461505133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:51:04.464866244Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:51:05.246113297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:51:05.249580603Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:51:05.974677786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:51:05.978210324Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T18:51:06.55448615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:07.036510087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:51:07.03992662Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:51:07.804570981Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:51:09.104178443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:09.11188553Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:09.111911207Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:51:09.111939505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:09.120372279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:51:11.554496763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:11.554575735Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=info ts=2026-02-16T18:51:12.812936422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:51:12.817320765Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:51:13.417307307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:51:15.229318445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:51:15.233957074Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:51:15.305100625Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-02-16T18:51:16.553623832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:16.61790279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:51:16.622084977Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:51:18.232936672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:51:19.054602815Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-02-16T18:51:19.054627449Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=15
level=warn ts=2026-02-16T18:51:21.554515117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:22.805132237Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:51:22.887064502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:51:22.891001276Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:51:25.888962249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:25.89834269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:51:26.553517701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:51:26.553546751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:51:26.553558551Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:51:26.553621858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:26.554502698Z caller=checkpoint.go:390 msg="flushing series" totalSize="226 kB" series=1
level=info ts=2026-02-16T18:51:26.559260541Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-02-16T18:51:26.560224059Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00581468s
level=info ts=2026-02-16T18:51:26.983853989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:51:26.989430635Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:51:28.303914053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" idle=1 full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:51:28.307873397Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:51:29.181158945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:29.186570775Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:29.186600758Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:51:29.186623737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:29.189886981Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:51:30.412007441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" idle=1 full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:51:30.415822512Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:51:30.503637857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:51:31.554598227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:33.294513565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:51:33.297913985Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:51:35.520351055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:51:35.523959151Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:51:36.520972362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:51:36.524516858Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:51:36.553641553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:37.484791744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:51:37.48830334Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:51:40.247389872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:51:40.251053742Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:51:41.042463366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:51:41.04694059Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:51:41.554296583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:42.09692734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="438 kB" avg_comp="146 kB" total_uncomp="4.4 MB" avg_uncomp="1.5 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:51:42.101164042Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:51:42.198849702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:51:42.20227899Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:51:42.206345196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:51:42.209443127Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:51:42.780363923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:51:42.783740646Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T18:51:43.418651169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:51:44.912594919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:51:44.916440085Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:51:46.554506036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:46.623597348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:51:46.628166824Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:51:47.615394597Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:51:51.216782306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:51:51.221503419Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:51:51.5544657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:51:52.127243242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:51:52.13143496Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:51:53.135326275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:51:53.140521124Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:51:53.99108581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:51:53.994633325Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T18:51:54.864220936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60400"
level=info ts=2026-02-16T18:51:56.535019695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T18:51:56.535035917Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:51:56.553357001Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:51:56.553528322Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-02-16T18:51:56.553548721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:51:56.553579283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:51:56.553590742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:51:56.553623109Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:51:56.553882267Z caller=checkpoint.go:378 msg="writing series" size="343 kB"
level=info ts=2026-02-16T18:51:58.289986381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:51:58.298034342Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:51:59.742500589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:51:59.750994376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:51:59.879236079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:51:59.883464518Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:52:00.309844044Z caller=checkpoint.go:378 msg="writing series" size="8.7 MB"
level=debug ts=2026-02-16T18:52:00.309872147Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.0 MB" series=2
level=warn ts=2026-02-16T18:52:01.553856347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:01.988316308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:52:01.991844357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:52:04.054686595Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:52:05.525984956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:52:05.530389102Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:52:06.553760114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:06.622965909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:52:06.626902956Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:52:07.804066956Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-02-16T18:52:11.387664248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:52:11.391978017Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T18:52:11.554407873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:11.554553649Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=debug ts=2026-02-16T18:52:11.684589415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:52:13.095533043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:52:13.099130211Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:52:13.419593069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:52:15.169054252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:52:15.172709332Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:52:15.304731717Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T18:52:15.30476091Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-02-16T18:52:15.593670451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:52:15.597367006Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:52:16.459993229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:52:16.55457141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:17.735094477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:52:17.740023454Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:52:19.054169789Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=warn ts=2026-02-16T18:52:21.553928124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:22.086436631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:52:22.090080507Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:52:22.09010481Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:52:22.090127212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:52:22.093365359Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:52:22.804263628Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=info ts=2026-02-16T18:52:25.349932747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:52:25.353440333Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:52:26.554351829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:52:26.554381395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:52:26.554390175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:52:26.554387759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:26.554450745Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:52:28.710552537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:52:28.714839357Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:52:30.305024208Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-02-16T18:52:30.305049419Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-02-16T18:52:30.468778632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:52:30.472551329Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:52:31.49817885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:52:31.503340259Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:52:31.554468557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:34.054275403Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T18:52:34.365387799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:52:34.369182946Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:52:34.525535045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:52:34.529330614Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:52:36.509819873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:52:36.514605009Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:52:36.553663944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:36.562969476Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:52:37.092500829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:52:37.096693003Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:52:37.804090032Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T18:52:38.006628106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:52:38.01088906Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:52:39.915824415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:52:39.919790595Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T18:52:40.013183984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="438 kB" avg_comp="146 kB" total_uncomp="4.4 MB" avg_uncomp="1.5 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:52:40.017602014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:52:41.553934943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:52:41.553915716Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
ts=2026-02-16T18:52:43.421512219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:52:45.305267341Z caller=checkpoint.go:378 msg="writing series" size="925 kB"
level=debug ts=2026-02-16T18:52:45.305295942Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T18:52:45.694425253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:52:45.700652192Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:52:46.55422678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:47.185769892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:52:47.189609344Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:52:47.59200169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:52:47.598235092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:52:49.054396561Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T18:52:51.554539208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:52.11625201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:52:52.12056124Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:52:52.596137121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:52:52.598629084Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:52:52.804660142Z caller=checkpoint.go:378 msg="writing series" size="438 kB"
level=info ts=2026-02-16T18:52:53.635447388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:52:53.639090107Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:52:53.791524517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:52:53.797026586Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:52:54.575599849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:52:54.579489303Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:52:54.579510388Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:52:54.579532637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:52:54.583043539Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
ts=2026-02-16T18:52:54.866320825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58600"
level=info ts=2026-02-16T18:52:55.743038596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:52:55.748268052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:52:55.749723779Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:52:55.749727734Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:52:56.138935591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:52:56.142734224Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:52:56.533930368Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:52:56.53602778Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:52:56.554437141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:52:56.554486868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:52:56.554507895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T18:52:56.554508872Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:52:56.554519795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:53:00.308155126Z caller=checkpoint.go:378 msg="writing series" size="6.9 MB"
level=debug ts=2026-02-16T18:53:00.30818468Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.3 MB" series=4
ts=2026-02-16T18:53:01.1923553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54512"
level=warn ts=2026-02-16T18:53:01.554321711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:03.303385888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:53:03.308517684Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:53:04.054025795Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-02-16T18:53:04.067044117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:53:04.070711858Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:53:04.070734436Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:04.070756819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:53:04.074037827Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:53:06.000156308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:53:06.004733205Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:53:06.004759662Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:06.004789365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:53:06.00807691Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:53:06.553686861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:07.114553649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="365 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:53:07.118241945Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:53:07.804809776Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=debug ts=2026-02-16T18:53:08.888574768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:53:09.127412322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:53:09.131598817Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:53:11.553640496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:11.554736622Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T18:53:11.716594293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:11.720213851Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:53:13.423606684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:53:14.160529269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:53:14.164016144Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:53:14.643380378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:53:14.646813225Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:53:15.304576971Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:53:16.086067477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:53:16.090727452Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:53:16.554068466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:18.092649813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:53:18.097344194Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:53:19.054318611Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-02-16T18:53:20.466707528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:53:20.470564646Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:53:21.251801445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.1 kB" total_uncomp="70 kB" avg_uncomp="35 kB" full=2 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:53:21.25574293Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c679e60cd%3A19c679e60ce%3Ab8ef7503?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T18:53:21.255766928Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:21.255790187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="302 kB" avg_uncomp="302 kB" idle=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:53:21.259306367Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:53:21.536945946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:53:21.540351896Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:53:21.554415411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:22.804443906Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-02-16T18:53:24.836924764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:53:24.841128858Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:53:26.006390962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:53:26.011313396Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:53:26.554442295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:26.554442276Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:53:26.554459411Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=debug ts=2026-02-16T18:53:26.554493267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:53:26.554502047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:53:30.304172454Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T18:53:31.041463638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:53:31.045370386Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:53:31.436471051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:53:31.4402856Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T18:53:31.554554392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:34.0540999Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-02-16T18:53:34.41610297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:53:34.422203709Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:53:34.562009774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:53:36.355754901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:53:36.360376548Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T18:53:36.55360866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:37.14250788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:53:37.149374935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:53:37.532510489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:53:37.536759226Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:53:37.826788843Z caller=checkpoint.go:378 msg="writing series" size="28 MB"
level=debug ts=2026-02-16T18:53:37.826816567Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=10
level=info ts=2026-02-16T18:53:38.597917285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:53:38.601752886Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:53:38.626971617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="438 kB" avg_comp="146 kB" total_uncomp="4.4 MB" avg_uncomp="1.5 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:53:38.630884547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:53:41.554507527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:41.554554444Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-02-16T18:53:41.821306897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:53:41.825267079Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:53:42.079143877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:53:42.082902037Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:53:42.08292123Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:42.08294202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:53:42.086041006Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:53:42.942838183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:53:42.948134311Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:53:43.425599975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:53:43.457303617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:53:43.461001793Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:53:45.304763218Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T18:53:46.554188896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:47.925736021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:53:47.929959014Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:53:49.054764826Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-02-16T18:53:49.054788267Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-02-16T18:53:51.273876218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T18:53:51.553774616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:53:51.583246113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="438 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:53:51.586447299Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:53:51.667682105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:53:51.670962873Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:53:52.558530711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:53:52.561860291Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:53:52.80456032Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T18:53:52.82574672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:52.82919164Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:52.829212976Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:52.829235974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:53:52.832309364Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T18:53:53.362965495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51998"
level=info ts=2026-02-16T18:53:53.662249333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:53:53.665712541Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:53:55.503700633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:53:55.508997564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:53:55.509026607Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:53:55.509056268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:53:55.512238571Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:53:56.534071831Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:53:56.535205455Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:53:56.553592681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:53:56.553621892Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:53:56.55363373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:53:56.553687713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:53:56.554916325Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
level=info ts=2026-02-16T18:53:56.623583924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:53:56.626615039Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:53:58.427988394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:53:58.431371174Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:54:00.30413789Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T18:54:01.199462298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41700"
level=warn ts=2026-02-16T18:54:01.554611294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:01.675281358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:54:03.423168309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:54:03.426995792Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:54:04.054084103Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-02-16T18:54:06.55420249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:07.259599559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:54:07.268262835Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:54:07.803834731Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:54:08.105748019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:54:08.11034802Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:54:08.583973071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:54:08.588995229Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:54:11.554393471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:11.55454966Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
ts=2026-02-16T18:54:13.427064696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:54:13.644524923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:54:13.648617062Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:54:15.304388354Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T18:54:15.347755716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:54:15.351560104Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:54:16.553594045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:18.746812956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:54:18.751318272Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:54:19.054130722Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:54:20.056951105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:54:20.060936401Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:54:20.780814426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:54:20.785994816Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:54:20.786020655Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:20.786059191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:54:20.790236434Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T18:54:21.554437003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:21.899535367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:54:21.904035402Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:54:22.804561518Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
ts=2026-02-16T18:54:23.364834938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53862"
ts=2026-02-16T18:54:24.872366675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40540"
level=info ts=2026-02-16T18:54:26.554403936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:54:26.554402294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:26.554431705Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:54:26.554439723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:54:26.554502676Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=debug ts=2026-02-16T18:54:26.554516822Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T18:54:27.116404979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:54:27.120267611Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:54:27.498292838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:54:27.502179367Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T18:54:27.857509514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:54:30.304958391Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T18:54:30.84534301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:30.848737225Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:30.848756894Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:30.848778126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:30.851454135Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:54:31.554579375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:32.543236945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:54:32.547664985Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:54:33.445138966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:54:33.448521268Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:54:33.869766182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:54:33.876301522Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:54:33.905345188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:54:33.909877594Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:54:34.054351972Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T18:54:34.516748091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" idle=1 full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:34.52097829Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:34.521007243Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:34.521038315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:34.524546641Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:54:34.877604977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:34.882558351Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:34.882579744Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:34.882602238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:34.887025581Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:54:34.985418899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:54:34.988705493Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:54:35.535336214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:54:35.53946838Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:54:35.539520063Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:54:35.539550318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:54:35.543195758Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:54:36.444925867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:54:36.554190698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:37.16692525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:54:37.170330661Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:54:37.80444456Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T18:54:38.440805921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="442 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:54:38.446299768Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:54:38.487385159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:54:38.491488077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:54:41.547036377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:54:41.550178567Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:54:41.554274583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:41.554286223Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=info ts=2026-02-16T18:54:42.184963594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:54:42.188705991Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:54:43.427996662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:54:45.304100437Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T18:54:46.55420288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:54:47.13106247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:54:47.13659522Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:54:48.254403676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:54:48.259005433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:54:49.054417816Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=info ts=2026-02-16T18:54:49.213853284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:54:49.218301415Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:54:50.590133299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:54:50.593656658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:54:51.390022698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:54:51.394519487Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:54:51.55445381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:52.804189796Z caller=checkpoint.go:378 msg="writing series" size="763 B"
ts=2026-02-16T18:54:53.365876422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35456"
level=info ts=2026-02-16T18:54:54.666097042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:54:54.669737875Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:54:56.534135091Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:54:56.535203452Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:54:56.553568461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:54:56.553592511Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:54:56.553601015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:54:56.553606172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:54:56.554717422Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T18:54:59.218865315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:54:59.223582815Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:55:00.017078035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:00.020968063Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:00.020989234Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:00.021011974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:00.024390428Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:55:00.304652606Z caller=checkpoint.go:378 msg="writing series" size="654 kB"
level=debug ts=2026-02-16T18:55:00.304678294Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-02-16T18:55:00.475945881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:55:00.479995784Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:55:01.049167937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:55:01.052834105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T18:55:01.554111304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:01.745237127Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T18:55:04.054882774Z caller=checkpoint.go:378 msg="writing series" size="477 kB"
level=info ts=2026-02-16T18:55:05.573677152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:55:05.577575638Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:55:06.553618852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:06.942996329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:06.947701177Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:06.947719153Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:06.947739701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:06.95217518Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:55:07.803951699Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=info ts=2026-02-16T18:55:10.645379749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:55:10.650864182Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:55:10.801282436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:55:10.805202826Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:55:11.554368358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:11.554784313Z caller=checkpoint.go:378 msg="writing series" size="955 kB"
level=debug ts=2026-02-16T18:55:11.554802062Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=3
ts=2026-02-16T18:55:13.429805365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:55:13.498535725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:55:13.502548864Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:55:13.778532942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:55:13.782348931Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:55:13.782368017Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:13.782388045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:55:13.785593237Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:55:15.304195385Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T18:55:16.416130174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:55:16.423181323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:55:16.553785253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:17.202530979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:17.20650136Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:55:19.054330502Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T18:55:19.452663033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="528 B" avg_comp="264 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:55:19.455981552Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:55:19.979618023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:55:19.984649419Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:55:20.629071408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:55:20.632454679Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:55:20.989397349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:20.994616275Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:55:21.256395424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:55:21.259875385Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:55:21.259896019Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:21.259917305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:55:21.262844985Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T18:55:21.554295032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:22.192440844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="442 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:55:22.196502216Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:55:22.196524281Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:22.196547994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="442 B" total_uncomp="1.4 kB" avg_uncomp="339 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:55:22.19963348Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:55:22.80636748Z caller=checkpoint.go:378 msg="writing series" size="3.0 MB"
level=debug ts=2026-02-16T18:55:22.806389478Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=3
level=info ts=2026-02-16T18:55:23.602504905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:55:23.606701646Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:55:24.175522791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:55:24.180179482Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:55:24.307411569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:24.311122167Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:24.31114525Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:24.311170631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:24.314265591Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:55:24.821952537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:55:24.825744729Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T18:55:24.876383443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51602"
level=info ts=2026-02-16T18:55:26.55369256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:55:26.553723699Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:55:26.553734542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:55:26.553726507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:26.553959487Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T18:55:26.928378401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:55:26.931663193Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:55:28.737389886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:55:30.304528827Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T18:55:31.554201435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:33.045289256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:55:33.049517486Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:55:33.371422443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:55:33.374939456Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T18:55:34.054160758Z caller=checkpoint.go:378 msg="writing series" size="401 kB"
level=debug ts=2026-02-16T18:55:35.824463674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:55:36.553691294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:36.855384833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:55:36.85893645Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:55:37.804804862Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T18:55:37.827036836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:55:37.83051553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T18:55:37.830534645Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:37.830558484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:55:37.833369942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:55:37.84245062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:55:37.845443286Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:55:38.812303213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:55:38.815513107Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:55:41.018533534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:55:41.023711811Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:55:41.417794212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" idle=1 full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:55:41.421640296Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:55:41.554113284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:41.554138563Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T18:55:41.710636653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:55:41.7156133Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T18:55:43.431561685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:55:45.304006881Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T18:55:45.551034929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:55:45.554572788Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:55:45.914384651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:55:45.917763188Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:55:46.017163332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:55:46.020613953Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T18:55:46.554489491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:55:49.054225194Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-02-16T18:55:51.553854086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:51.823843229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:55:51.827318557Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:55:52.804567433Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T18:55:53.331329719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:55:53.335224485Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:55:53.335246327Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:55:53.33527303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:55:53.338337143Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:55:54.226055505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:55:54.229762442Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:55:56.534369411Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:55:56.535408868Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:55:56.553723177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:55:56.553754439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:55:56.55376317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:55:56.553809817Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T18:55:56.553843567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:55:56.707868678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:55:56.712846022Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:55:58.052598205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:55:58.057072582Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:55:59.346309095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:55:59.349721665Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:56:00.304776176Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-02-16T18:56:01.216695817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36550"
level=warn ts=2026-02-16T18:56:01.553790528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:03.420854005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:56:03.426458013Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:56:04.054511156Z caller=checkpoint.go:378 msg="writing series" size="170 kB"
level=debug ts=2026-02-16T18:56:04.054539054Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T18:56:05.358534711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:56:05.362507492Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T18:56:06.553853978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:07.804137514Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T18:56:08.346822553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="646 B" avg_comp="323 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:56:08.350898011Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:56:08.792495631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:56:08.798537842Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:56:10.437606963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:56:10.442195129Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:56:10.664593345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:56:11.11481912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:56:11.12015863Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:56:11.553783781Z caller=checkpoint.go:378 msg="writing series" size="6.3 kB"
level=warn ts=2026-02-16T18:56:11.553793553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:12.011928141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:56:12.018734406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:56:13.116353476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:56:13.12038098Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:56:13.120401839Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:56:13.120427321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:56:13.124199311Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:56:13.357854206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:56:13.434337665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:56:14.562535841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:56:14.56649968Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T18:56:14.661683008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:56:14.665450702Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:56:15.304214686Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-02-16T18:56:16.553627809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:19.054508029Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T18:56:19.888955929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:56:19.8930388Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T18:56:21.154836145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:56:21.158102301Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:56:21.554102577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:22.46656588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:56:22.470058683Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:56:22.804291742Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T18:56:23.782379125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:56:23.786065319Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T18:56:23.998622741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:56:24.002116317Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:56:26.553522287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:56:26.553557952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:56:26.553568758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:56:26.553605643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:26.554627827Z caller=checkpoint.go:390 msg="flushing series" totalSize="272 kB" series=5
level=info ts=2026-02-16T18:56:26.560525761Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-02-16T18:56:26.561627128Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007912245s
level=info ts=2026-02-16T18:56:27.004529413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:56:27.009591373Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:56:27.131469416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:56:27.134797956Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:56:27.456360945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:56:27.45974663Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:56:30.713411738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:56:30.717385162Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T18:56:31.220227653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47070"
level=warn ts=2026-02-16T18:56:31.553809412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:33.545771104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:56:33.550251195Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T18:56:36.553996491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:36.805769126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:56:36.809550383Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T18:56:38.919558578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:56:40.876901858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:56:40.8806081Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T18:56:41.554008074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T18:56:43.437843934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:56:43.999897167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:56:44.005488205Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:56:44.225038549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:56:44.228515358Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:56:44.719870936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:56:44.723274689Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:56:46.553621544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:47.442336282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:56:47.445837247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T18:56:51.326591348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:56:51.330325825Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:56:51.553661851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:56:54.060885728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:56:54.066224248Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T18:56:54.880157692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43262"
level=info ts=2026-02-16T18:56:56.20570575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:56:56.210358722Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:56:56.534069982Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:56:56.535137157Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:56:56.553578153Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T18:56:56.553634555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:56:56.553645565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:56:56.553659418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:56:56.553667755Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:56:56.553722901Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-02-16T18:56:56.554067299Z caller=checkpoint.go:378 msg="writing series" size="482 kB"
level=info ts=2026-02-16T18:56:58.109803174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:56:58.113797342Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:56:59.619609359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:56:59.622872183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T18:57:00.309195484Z caller=checkpoint.go:378 msg="writing series" size="8.9 MB"
level=debug ts=2026-02-16T18:57:00.309223167Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.4 MB" series=2
level=warn ts=2026-02-16T18:57:01.553965689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:04.054399643Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T18:57:04.435788263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:57:04.441284888Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:57:04.825265135Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:57:05.611616904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:57:05.615310559Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:57:06.554120493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:07.225615778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:57:07.230139316Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:57:07.804824358Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-02-16T18:57:08.345435523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:57:08.34941091Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:57:08.468862107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:57:08.470579966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:57:10.093783089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:57:10.0975577Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:57:10.717449799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:10.724957054Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:57:11.553996655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:11.554100495Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=info ts=2026-02-16T18:57:11.939656209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:57:11.943304254Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T18:57:12.290445826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:57:12.293940864Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T18:57:13.441437611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T18:57:14.250323274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:57:14.253728061Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T18:57:15.305911551Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T18:57:15.305939327Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-02-16T18:57:16.554413564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:19.054405699Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-02-16T18:57:19.520743225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:57:19.52471192Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:57:19.993134316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:57:19.996938323Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:57:20.63070348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:57:20.636413885Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T18:57:21.553609218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:21.723995673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:57:21.727783031Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T18:57:22.804692643Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=info ts=2026-02-16T18:57:24.791282629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:57:24.794748536Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:57:26.166595024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:57:26.170143129Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:57:26.554235746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:57:26.554245466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:26.554268516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:57:26.554276686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:57:26.554303087Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-02-16T18:57:30.305743942Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-02-16T18:57:30.305770618Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=4
level=info ts=2026-02-16T18:57:30.90306163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:57:30.906590689Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T18:57:31.554461024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:32.853400879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:57:32.856955733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:57:33.699400603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:57:33.703143048Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:57:34.054328814Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-02-16T18:57:36.554272218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:37.804133265Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T18:57:38.255772909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:57:39.216499751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:57:39.220888749Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T18:57:41.554000469Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=warn ts=2026-02-16T18:57:41.554015195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:42.028608155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:57:42.032249761Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:57:42.162847978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:57:42.16902708Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T18:57:42.627105655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:57:42.63082367Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:57:42.630845668Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:57:42.63086666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:57:42.633745105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:57:43.443244315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:57:45.304821026Z caller=checkpoint.go:378 msg="writing series" size="789 kB"
level=debug ts=2026-02-16T18:57:45.304844255Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-02-16T18:57:46.554265613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:47.07919151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:57:47.083370467Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:57:48.195208357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:57:49.054904404Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T18:57:51.140371481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:57:51.144094601Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T18:57:51.554548973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:57:52.20385386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:57:52.207361226Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T18:57:52.804367182Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
level=info ts=2026-02-16T18:57:52.975743148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:57:52.979122768Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T18:57:53.381201402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43074"
level=info ts=2026-02-16T18:57:55.169239477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:57:55.173111714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:57:56.206614382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:57:56.212061738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:57:56.53412461Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:57:56.535168436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:57:56.553517123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:57:56.553544473Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:57:56.553558129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:57:56.553596301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:57:56.554656078Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T18:57:57.581370985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:57.588131674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:57.588154165Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:57:57.58818201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:57:57.595086692Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:57:58.954297936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:57:58.9583653Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:57:58.958389766Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:57:58.958413316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:57:58.961656449Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:57:59.038998568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:57:59.043632241Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T18:57:59.644419039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:57:59.647854109Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:58:00.139830046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:58:00.143522788Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T18:58:00.310435599Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T18:58:00.310458502Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.5 MB" series=4
level=warn ts=2026-02-16T18:58:01.553849574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:01.775448079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:58:01.779957188Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T18:58:04.054964854Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=info ts=2026-02-16T18:58:04.945797696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:58:04.950805713Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:58:06.382525071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:58:06.385796066Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T18:58:06.55443974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:07.805037045Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=info ts=2026-02-16T18:58:09.195363917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:09.198704117Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:58:09.765530096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:58:09.770125543Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T18:58:10.344106704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:58:11.318177172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:58:11.325313752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T18:58:11.553746653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:11.554880087Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-02-16T18:58:12.443106293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:58:12.446851607Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:58:13.311611068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:58:13.314955523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
ts=2026-02-16T18:58:13.444941549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:58:14.330104935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:58:14.333213046Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:58:14.906896574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:58:14.910420448Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:58:15.304537095Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T18:58:16.554338832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:19.054896021Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-02-16T18:58:20.896326522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:58:20.901223038Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:58:21.553946438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:22.804051805Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=info ts=2026-02-16T18:58:24.092882384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="375 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:58:24.096598299Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T18:58:25.297808096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:58:25.30137719Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T18:58:25.561129391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:58:25.564957978Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T18:58:26.554451545Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T18:58:26.554451634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T18:58:26.554463617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:26.554494022Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:58:26.554505536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T18:58:26.556630081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:58:26.560407053Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T18:58:29.262931723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:58:29.267415595Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T18:58:30.284307116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:58:30.289901209Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:58:30.304000399Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-02-16T18:58:31.232422386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42058"
level=warn ts=2026-02-16T18:58:31.554448929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:32.065306675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:58:32.073611713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T18:58:34.054501466Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-02-16T18:58:34.324451663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:58:34.328617921Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:58:34.735943848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:58:34.739534922Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:58:34.863756163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:58:34.867389145Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T18:58:36.554279862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:37.825030038Z caller=checkpoint.go:378 msg="writing series" size="29 MB"
level=debug ts=2026-02-16T18:58:37.825054915Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=10
level=debug ts=2026-02-16T18:58:38.320616321Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T18:58:41.553611211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:41.554748483Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
ts=2026-02-16T18:58:43.446054279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:58:43.660867835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T18:58:44.084041455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:58:44.087404703Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T18:58:45.304838794Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-02-16T18:58:46.350246641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:58:46.353632993Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:58:46.554388434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:49.055081534Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-02-16T18:58:49.055106607Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-02-16T18:58:51.553699588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:58:52.274955413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:58:52.281320725Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T18:58:52.804336264Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T18:58:53.010536806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:58:53.014440885Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:58:53.776101928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:58:53.779394103Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:58:53.779416301Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:58:53.779438088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:58:53.782265165Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T18:58:55.155534342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:58:55.160648199Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T18:58:56.53452216Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:58:56.535573972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T18:58:56.553890992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:58:56.5539195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:58:56.553930066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:58:56.553956533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:58:56.554635521Z caller=checkpoint.go:378 msg="writing series" size="541 kB"
level=info ts=2026-02-16T18:58:57.335579344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:58:57.339173127Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:58:58.126642913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:58:58.130079828Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:58:58.871334644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:58:58.875678645Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:59:00.304445874Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:59:00.384889304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:59:00.388603384Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:59:00.948358692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:59:00.951812922Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T18:59:01.554378019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:04.054022Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=info ts=2026-02-16T18:59:04.304829832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:59:04.308591931Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T18:59:04.645882332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:59:04.649353457Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T18:59:06.55452703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:07.804852931Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T18:59:09.612517819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:59:09.616380986Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:59:09.918227358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:59:09.922213197Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T18:59:11.554427531Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=warn ts=2026-02-16T18:59:11.554427083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:11.938693491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:59:11.94276486Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:59:12.07315726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:59:12.276946846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T18:59:12.280654475Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T18:59:13.206122377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:59:13.20986463Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:59:13.209889262Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:59:13.209912313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:59:13.213119811Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
ts=2026-02-16T18:59:13.448076355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=debug ts=2026-02-16T18:59:15.304154163Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-02-16T18:59:16.553810162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:16.858602836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T18:59:16.86226449Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T18:59:17.575188139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T18:59:17.579470405Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T18:59:19.054820183Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T18:59:20.064660052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T18:59:20.068246884Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T18:59:21.554398873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:22.050308407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T18:59:22.054103284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T18:59:22.804779396Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T18:59:23.264237688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T18:59:23.27077282Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T18:59:23.340066542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T18:59:23.343911845Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T18:59:24.892369358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60338"
level=info ts=2026-02-16T18:59:25.637363616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="57 MB" avg_uncomp="19 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T18:59:25.643758414Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T18:59:26.554198833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:59:26.55422734Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:59:26.554234871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T18:59:26.554251844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:26.554304679Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=debug ts=2026-02-16T18:59:26.554320627Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T18:59:26.833315406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T18:59:26.838426274Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T18:59:30.304336713Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T18:59:30.472930862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T18:59:30.476587961Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T18:59:31.554533666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:31.766216048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T18:59:31.76967402Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T18:59:33.851832612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T18:59:34.054380542Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T18:59:34.931522278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T18:59:34.934671818Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T18:59:35.270505488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T18:59:35.274324082Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T18:59:36.554111753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:36.906603444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T18:59:36.910511437Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T18:59:37.804472263Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T18:59:38.643084278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T18:59:38.647925207Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T18:59:40.118155873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:59:40.12161298Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:59:40.121634892Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:59:40.121659409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:59:40.124578119Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T18:59:41.553884329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:41.553995988Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-02-16T18:59:42.779072719Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T18:59:42.828235652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:59:42.831665527Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:59:42.831684617Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:59:42.831706707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:59:42.834543364Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T18:59:43.450549885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T18:59:44.107278777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T18:59:44.111121639Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T18:59:45.070700613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T18:59:45.074412096Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T18:59:45.304120473Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T18:59:46.553577751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:48.612779529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T18:59:48.616250994Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T18:59:49.054852537Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=warn ts=2026-02-16T18:59:51.553835806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T18:59:52.804951203Z caller=checkpoint.go:378 msg="writing series" size="763 B"
ts=2026-02-16T18:59:53.390399616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47090"
level=info ts=2026-02-16T18:59:53.920465237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T18:59:53.928174017Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T18:59:54.21578221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T18:59:54.219365811Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T18:59:54.455102369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T18:59:54.459042044Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T18:59:55.733710003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T18:59:55.738792873Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T18:59:56.533771731Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T18:59:56.535900224Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T18:59:56.554278971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T18:59:56.554285233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T18:59:56.554309794Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T18:59:56.554320888Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T18:59:56.554402897Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T18:59:58.203638657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T18:59:58.206973691Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T18:59:58.578264816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:59:58.581751454Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:59:58.581772041Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T18:59:58.581793986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="45 kB" avg_comp="45 kB" total_uncomp="302 kB" avg_uncomp="302 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T18:59:58.584718903Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T18:59:58.858494274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T18:59:58.861860013Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:00:00.304670029Z caller=checkpoint.go:378 msg="writing series" size="626 kB"
level=debug ts=2026-02-16T19:00:00.304695538Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=9
level=info ts=2026-02-16T19:00:00.458792844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:00:00.463191218Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T19:00:01.554371087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:04.054448448Z caller=checkpoint.go:378 msg="writing series" size="496 kB"
level=warn ts=2026-02-16T19:00:06.554596223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:07.753596762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:00:07.757190545Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T19:00:07.804681455Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T19:00:08.691461338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:00:08.694960614Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:00:09.218792563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:00:09.222832125Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:00:10.817297165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" idle=1 full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:00:10.820855102Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:00:10.901085792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:00:10.904684598Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:00:11.522375977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:00:11.528902194Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T19:00:11.554101138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:11.554937163Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:00:11.554953307Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=3
level=debug ts=2026-02-16T19:00:11.683526989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:00:12.952256368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:00:12.956252251Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T19:00:13.452046562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:00:13.677397068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:00:13.681045132Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:00:13.767439548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:00:13.773333209Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:00:15.305056254Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T19:00:15.645075713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:00:15.648709965Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T19:00:16.553595545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:19.054072423Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T19:00:19.424199882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="664 B" avg_comp="332 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:00:19.428047629Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:00:21.554521402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:22.01319182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:00:22.016697488Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:00:22.016719174Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:22.016742689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:00:22.019896019Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:00:22.542667402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:00:22.54776376Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:00:22.807199137Z caller=checkpoint.go:378 msg="writing series" size="2.8 MB"
level=debug ts=2026-02-16T19:00:22.807224087Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=3
ts=2026-02-16T19:00:23.39175861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46862"
level=info ts=2026-02-16T19:00:26.553590028Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:00:26.553622398Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:00:26.553640333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:00:26.553643235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:26.555512719Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=debug ts=2026-02-16T19:00:26.87389715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:00:27.177535624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:00:27.181643774Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T19:00:30.304093201Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=warn ts=2026-02-16T19:00:31.554304372Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:31.586464474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:00:31.590114368Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:00:33.14580577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:00:33.149312289Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:00:33.399968241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="438 kB" avg_comp="110 kB" total_uncomp="4.4 MB" avg_uncomp="1.1 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:00:33.404325355Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:00:33.4204043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:00:33.423785436Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:00:34.054595929Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=info ts=2026-02-16T19:00:34.434423657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:00:34.438161993Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:00:36.554056556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:00:36.926001132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:00:36.929736373Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:00:37.805153709Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T19:00:38.855365548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:00:38.859232484Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:00:39.057763915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:39.061123834Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:39.061148802Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:39.061173171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:39.06398486Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:00:39.593635491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:00:39.596984831Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:00:41.515810775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T19:00:41.553697979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:41.554787438Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T19:00:42.268026159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:00:42.2718031Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T19:00:43.453346639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:00:44.349902049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:00:44.358878209Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T19:00:45.304397231Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T19:00:45.525209203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:00:45.529936684Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:00:45.529961582Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:45.529985983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:00:45.534633344Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T19:00:46.553562029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:49.054099732Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T19:00:50.019494819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:00:50.024506027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:00:50.399602564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:00:50.403547069Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:00:51.553830555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:52.804810932Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
ts=2026-02-16T19:00:53.392911566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55674"
level=info ts=2026-02-16T19:00:54.897111837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:00:54.902344903Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:00:56.356972363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:00:56.361346086Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:00:56.511725287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:00:56.515139988Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:00:56.515161439Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:00:56.515181816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:00:56.518253003Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:00:56.534069486Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:00:56.535123844Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T19:00:56.536451114Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T19:00:56.536528159Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968075 n=1
level=debug ts=2026-02-16T19:00:56.536573228Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968075 n=1
level=info ts=2026-02-16T19:00:56.553501596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:00:56.553527399Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:00:56.553545308Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:00:56.553644534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:00:56.55470318Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T19:00:58.62169294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:00:58.628052731Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:00:58.978042058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:00:58.982786392Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:01:00.304549081Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T19:01:00.480120921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:00.485962267Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:00.485984306Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:00.486010292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:00.4912281Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T19:01:01.248579838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46666"
level=warn ts=2026-02-16T19:01:01.55400389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:01.639340364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:01:01.643935167Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:01:02.024296781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:01:02.027941625Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:01:02.141685562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:01:02.145255305Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T19:01:04.054869199Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=info ts=2026-02-16T19:01:06.216171852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:01:06.220246961Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T19:01:06.553783364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:06.766640015Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:01:07.586457467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:07.591390717Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:07.591416602Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:07.591450902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:07.594631956Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:01:07.804343465Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T19:01:11.55443666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:11.554439477Z caller=checkpoint.go:378 msg="writing series" size="8.5 kB"
level=debug ts=2026-02-16T19:01:11.554462491Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-02-16T19:01:13.454463788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T19:01:14.750508856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" idle=1 full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:14.755734323Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:14.755757785Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:14.755783044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:14.759270793Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:01:15.252674282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:01:15.256575075Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:01:15.304663155Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-02-16T19:01:16.28119629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:16.286523251Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:01:16.549362525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:01:16.552658608Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T19:01:16.553657974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:17.735988608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:01:17.737707844Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T19:01:17.776841273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:01:17.781607938Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:01:17.781625338Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:17.781647302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:01:17.785652083Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:01:18.589994862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="839 B" avg_comp="279 B" total_uncomp="732 B" avg_uncomp="244 B" idle=1 full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:01:18.593756255Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:01:18.809391498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:18.812954992Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:01:19.054758778Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T19:01:21.553769104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:22.478912226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:01:22.482400409Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:01:22.638160984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:01:22.641548133Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:01:22.778909524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:22.782230803Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:22.782252463Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:22.782272239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:22.785032812Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T19:01:22.804273278Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
ts=2026-02-16T19:01:23.394006396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34648"
level=info ts=2026-02-16T19:01:24.418156042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:01:24.421428572Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:01:25.545674547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:01:25.550298959Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:01:26.554144676Z caller=checkpoint.go:390 msg="flushing series" totalSize="266 kB" series=3
level=warn ts=2026-02-16T19:01:26.554172122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:26.554188323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:01:26.554206036Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:01:26.554215625Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:01:26.562173095Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-02-16T19:01:26.563185399Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009307796s
level=info ts=2026-02-16T19:01:29.204121882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:01:29.207713004Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:01:31.553661725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:33.133099858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:33.136604771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:01:33.458340206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:33.4621903Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:01:34.796142229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:01:34.79961166Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:01:34.799632339Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:34.799656419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:01:34.802549302Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:01:35.170600543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="45 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:01:35.173985883Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:01:36.340614408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:01:36.344049955Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:01:36.346096885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:01:36.350295695Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T19:01:36.554047713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:37.01815639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:01:37.022807559Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:01:37.213380847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:01:38.05140365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:01:38.056299701Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:01:38.056323124Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:38.056352116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:01:38.060139732Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T19:01:41.55436655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T19:01:43.457067708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T19:01:43.878709485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:01:43.883511143Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:01:44.089184859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:01:44.093174861Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:01:44.471264282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:01:44.475271234Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:01:45.066033242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:01:45.072118915Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:01:46.553996826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:47.007988615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:01:47.011289419Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:01:49.252742381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:01:49.257061987Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T19:01:50.431255902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:01:50.438154255Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:01:50.972790766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:01:50.976931536Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T19:01:51.55369524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:01:52.60572554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:01:52.60926022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:01:52.655421865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:01:52.658914168Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:01:54.519036618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:01:54.522423508Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:01:55.542454299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:01:55.545662443Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:01:56.450916504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:01:56.455198277Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:01:56.533983049Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:01:56.536056777Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:01:56.55335525Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T19:01:56.553655289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:01:56.553663815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:01:56.553676849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:01:56.553686264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:01:56.553702063Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=debug ts=2026-02-16T19:01:56.554029428Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
level=info ts=2026-02-16T19:01:56.933119421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:01:56.936351981Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:01:56.936370007Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:01:56.936391871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:01:56.939130124Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T19:02:00.311812985Z caller=checkpoint.go:378 msg="writing series" size="9.7 MB"
level=debug ts=2026-02-16T19:02:00.311838314Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=2
level=warn ts=2026-02-16T19:02:01.554229266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:04.054416736Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T19:02:04.142046605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:02:04.145535365Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:02:06.553928544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:07.610737536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:02:07.614672912Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:02:07.804361875Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-02-16T19:02:08.24271082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:08.246133242Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:08.246157267Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:08.246180157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:08.24956487Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:02:10.107174593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:10.112058141Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:02:10.278725055Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:02:11.280662763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:11.284286775Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:02:11.425806041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:02:11.42938064Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:02:11.554533055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:11.55466199Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=info ts=2026-02-16T19:02:13.302603953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:02:13.305918559Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:02:13.305943001Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:13.305965517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:02:13.308850438Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T19:02:13.459667697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T19:02:15.305355872Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:02:15.305379716Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-02-16T19:02:15.493590131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T19:02:16.553924594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:16.575072524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:16.578312139Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:02:17.168760745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:02:17.172889619Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:02:19.054637392Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-02-16T19:02:19.306275514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:02:19.311795357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:02:19.697202752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:02:19.700648887Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:02:19.80902949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:02:19.812624752Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T19:02:21.554173648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:22.804926284Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
ts=2026-02-16T19:02:23.396123916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44502"
level=info ts=2026-02-16T19:02:24.163148027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:24.166508069Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:02:26.554518087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:02:26.554546862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:02:26.554555368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:02:26.554574373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:26.55457276Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T19:02:27.630338442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:02:27.634599654Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:02:29.097659699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:02:29.101172616Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:02:30.305507549Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB"
level=debug ts=2026-02-16T19:02:30.305530795Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-02-16T19:02:30.49933417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:02:30.503355821Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T19:02:30.503383557Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:30.503419199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:02:30.506321233Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T19:02:30.677941813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:02:30.681365481Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T19:02:31.258410692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40978"
level=warn ts=2026-02-16T19:02:31.554532712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:32.68735854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:02:32.691260564Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:02:32.69128074Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:32.691310024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:02:32.6941379Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:02:33.163687424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:02:33.170111198Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:02:33.581949506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:02:33.58668709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:02:34.054307626Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T19:02:35.507222969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:02:35.511016606Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T19:02:36.554436423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:36.669005069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:02:36.672322837Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:02:36.672345306Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:02:36.672367248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:02:36.675240081Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:02:37.703345499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:02:37.706994043Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:02:37.804498063Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=warn ts=2026-02-16T19:02:41.553757589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:41.554885553Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=info ts=2026-02-16T19:02:42.787445073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:02:42.791127155Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T19:02:43.462048716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T19:02:43.809612289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:02:43.813552289Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:02:45.146021335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:02:45.15010458Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:02:45.305069632Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:02:45.305100413Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-02-16T19:02:46.218359632Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:02:46.32463317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:02:46.329580963Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:02:46.553579851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:46.971153593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:02:46.97472666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:02:49.054154003Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T19:02:49.099305767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:02:49.102990288Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:02:50.080379697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:02:50.083758368Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T19:02:50.70370384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:02:50.832291776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:02:50.835682508Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:02:51.554362565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:02:51.612509842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:02:51.616029484Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:02:52.804853222Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
level=info ts=2026-02-16T19:02:54.877785212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:02:54.882609566Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T19:02:54.909437196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57608"
level=info ts=2026-02-16T19:02:56.312816447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:02:56.316588837Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:02:56.534102606Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:02:56.53515679Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:02:56.553521066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:02:56.55354324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:02:56.553559027Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:02:56.553560722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:02:56.554677356Z caller=checkpoint.go:378 msg="writing series" size="7.3 kB"
level=info ts=2026-02-16T19:02:58.17328987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:02:58.177321465Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:02:58.528430548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:02:58.531947031Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:02:59.383416296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:02:59.386988307Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:03:00.308900412Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T19:03:00.308926897Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.5 MB" series=4
level=info ts=2026-02-16T19:03:00.64861637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:03:00.652904672Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T19:03:01.262758704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:03:01.26602565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:03:01.553924963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:01.998868012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:03:02.002712028Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:03:03.842744601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:03:03.847081708Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T19:03:04.054839139Z caller=checkpoint.go:378 msg="writing series" size="204 kB"
level=info ts=2026-02-16T19:03:05.888869333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="379 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:03:05.892655943Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T19:03:06.553593163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:07.80467095Z caller=checkpoint.go:378 msg="writing series" size="133 kB"
level=warn ts=2026-02-16T19:03:11.55392836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:11.553972126Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
ts=2026-02-16T19:03:13.46408634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T19:03:14.332538388Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:03:15.304177607Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T19:03:15.616334384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:03:15.620167725Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:03:16.247052182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:03:16.251031266Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T19:03:16.554033788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:19.054809393Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T19:03:19.066920029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:03:19.071894945Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:03:19.847242674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:03:19.851848739Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T19:03:20.541739902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:03:20.545138996Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T19:03:21.553593165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:22.804343299Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-02-16T19:03:23.312752959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:03:23.316637134Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T19:03:23.398740003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48716"
ts=2026-02-16T19:03:24.911243574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47830"
level=info ts=2026-02-16T19:03:25.473636493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:03:25.477201336Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:03:26.554440304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:03:26.554446504Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=warn ts=2026-02-16T19:03:26.554439333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:26.554468373Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:03:26.554492459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:03:27.415494215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:03:27.422146084Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:03:27.613114624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:03:27.616963715Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:03:28.107280837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:03:28.112418063Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:03:29.674085593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:03:29.678247164Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:03:30.304137317Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T19:03:30.433757999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:03:30.437354127Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:03:31.554380115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:34.054970277Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-02-16T19:03:36.221404089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:03:36.225570003Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:03:36.554576194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:37.824643467Z caller=checkpoint.go:378 msg="writing series" size="29 MB"
level=debug ts=2026-02-16T19:03:37.824668393Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=10
level=info ts=2026-02-16T19:03:38.590673194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:03:38.59450886Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:03:38.594531609Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:03:38.594552618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:03:38.597236615Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:03:41.522165851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:03:41.527111917Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T19:03:41.554531103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:41.554603699Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-02-16T19:03:41.927737355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:03:41.931023944Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T19:03:43.465370951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T19:03:43.870379634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:03:43.873877073Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:03:44.107594104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:03:44.148903047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:03:44.152435735Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:03:44.454288427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:03:44.457758562Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:03:45.304360648Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T19:03:46.553855232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:49.054796414Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:03:49.054822653Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-02-16T19:03:49.492656267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:03:49.497179231Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:03:51.554445229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:03:52.481766417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:03:52.564197366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:03:52.567590414Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:03:52.576640461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="2.2 kB" avg_comp="439 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:03:52.579637551Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:03:52.71991563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:03:52.724018952Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T19:03:52.804540648Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T19:03:53.40724213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:03:53.410593789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:03:54.280002744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:03:54.284719371Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-02-16T19:03:54.912108628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60826"
level=debug ts=2026-02-16T19:03:56.534577972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:03:56.53563059Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:03:56.554010363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:03:56.554034682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T19:03:56.55403034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:03:56.554047466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:03:56.554292922Z caller=checkpoint.go:378 msg="writing series" size="403 kB"
level=info ts=2026-02-16T19:03:58.032750856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:03:58.037039472Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:04:00.303958588Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T19:04:01.059282111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:01.063623405Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T19:04:01.554355216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:04.054589313Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=warn ts=2026-02-16T19:04:06.554424553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:07.734219545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:04:07.73865274Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T19:04:07.804756746Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T19:04:08.10872562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="994 kB" avg_comp="994 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:04:08.108754365Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T19:04:08.113628565Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:04:08.765832612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:04:08.769567952Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:04:09.197680428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:04:09.201190572Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T19:04:10.60826952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:10.614919196Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:10.6149426Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:04:10.614969874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:10.620948983Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:04:11.158593477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:04:11.162172012Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:04:11.243427631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:04:11.24709496Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:04:11.553845125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:11.55405271Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-02-16T19:04:12.571719536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:12.575299188Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:12.575330578Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:04:12.575358016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:12.578409811Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T19:04:13.467130417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:04:14.673012378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:04:14.677213574Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:04:15.30399256Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=warn ts=2026-02-16T19:04:16.553977504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:17.641296345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:04:17.645030989Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T19:04:19.054710708Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T19:04:19.162101463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:04:19.165315524Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T19:04:21.554143879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:22.804839112Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T19:04:22.886141924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:22.89143552Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T19:04:23.402733022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:54648"
level=info ts=2026-02-16T19:04:23.531728303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:04:23.537760846Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:04:24.252676793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:04:24.258679107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T19:04:24.913555584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47076"
level=info ts=2026-02-16T19:04:25.300996234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:04:25.305902437Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:04:25.552977262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:04:25.556741416Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:04:26.554343206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:04:26.554374637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:04:26.554383347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:04:26.554379556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:26.554503204Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-02-16T19:04:27.212236138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:04:27.216017662Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:04:29.251285984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:04:29.254973912Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:04:29.733545016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:29.741002523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:04:30.304302256Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=debug ts=2026-02-16T19:04:30.304333585Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=11
ts=2026-02-16T19:04:31.270436441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39510"
level=warn ts=2026-02-16T19:04:31.554540955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:32.837751565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:04:32.843544799Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-02-16T19:04:34.054307046Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T19:04:34.496720816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:04:34.501943608Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T19:04:36.554260877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:37.804932948Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T19:04:38.033423934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:04:38.037303284Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:04:38.075438399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="2.2 kB" avg_comp="443 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:04:38.079301455Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:04:38.697223437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:04:41.188729168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:04:41.190893878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:04:41.192726067Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:04:41.192726602Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T19:04:41.553767672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:41.554926388Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
ts=2026-02-16T19:04:43.468929966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T19:04:45.304561107Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T19:04:46.146525178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:04:46.150783107Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T19:04:46.553966747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:04:47.023663833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:04:47.165777282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:04:47.16912864Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:04:47.499053312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:04:47.502315129Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:04:49.054036696Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=info ts=2026-02-16T19:04:49.387946847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:04:49.39142414Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:04:49.824818243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:04:49.82934134Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:04:50.297416587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:04:50.302055422Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:04:50.807792183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:04:50.811146362Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T19:04:51.554311273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:52.782066048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:04:52.785408524Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:04:52.804514928Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T19:04:54.709942578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:04:54.715244513Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:04:56.534457843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:04:56.53551065Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T19:04:56.553845364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:04:56.553886177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:04:56.553904498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:04:56.553912646Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:04:56.553912459Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T19:04:59.264326433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:04:59.26799659Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T19:04:59.470694375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:04:59.478759404Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:05:00.305201888Z caller=checkpoint.go:378 msg="writing series" size="632 kB"
level=debug ts=2026-02-16T19:05:00.305226033Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-02-16T19:05:00.999092254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:05:01.003043957Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:05:01.5539928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:03.244329782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="302 kB" avg_uncomp="151 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:05:03.248592465Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:05:03.715795743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:05:03.72036184Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=debug ts=2026-02-16T19:05:04.05476466Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=info ts=2026-02-16T19:05:05.820873399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:05:05.825316377Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:05:06.421732087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:05:06.427328465Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:05:06.55387695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:07.80486427Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=debug ts=2026-02-16T19:05:09.873397786Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T19:05:11.554254649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:11.554918059Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:05:11.554934782Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=3
ts=2026-02-16T19:05:13.470764889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:05:15.084366852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:05:15.088370248Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:05:15.304218713Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=warn ts=2026-02-16T19:05:16.554016226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:17.535174366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:05:17.540261319Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:05:17.64144668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:05:17.645633491Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:05:18.398663636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:05:18.402597141Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:05:19.054793921Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=info ts=2026-02-16T19:05:21.062232055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="2.2 kB" avg_comp="443 B" total_uncomp="1.7 kB" avg_uncomp="340 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:05:21.068528112Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T19:05:21.55446411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:22.792696285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:05:22.796174962Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:05:22.805974015Z caller=checkpoint.go:378 msg="writing series" size="2.9 MB"
level=debug ts=2026-02-16T19:05:22.805994921Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=3
ts=2026-02-16T19:05:24.915792488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54430"
level=info ts=2026-02-16T19:05:26.554264398Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:05:26.554292318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:05:26.554301214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:05:26.55426188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:26.554502569Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T19:05:26.893529452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:05:26.895510133Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:05:26.89553121Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:05:26.895557048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:05:26.897081527Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T19:05:29.286540241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:05:29.288026692Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T19:05:30.304271116Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T19:05:30.335326459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:05:30.339838329Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:05:31.51186686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="839 B" avg_comp="279 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:05:31.515219175Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:05:31.554227159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:05:32.570214926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:05:32.573943234Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:05:32.850758635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:05:32.854056877Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:05:33.442934506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:05:33.446955312Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T19:05:34.054446232Z caller=checkpoint.go:378 msg="writing series" size="457 kB"
level=info ts=2026-02-16T19:05:35.005211211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:05:35.008445822Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:05:36.553934955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:37.804184741Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T19:05:37.917405777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:05:37.922160704Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:05:39.787878188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:05:39.792094896Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:05:39.866446049Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:05:40.150541777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:05:40.154116892Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T19:05:41.553712836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:41.554792082Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T19:05:41.585917914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:05:41.589156157Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:05:41.93891913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:05:41.942179883Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T19:05:43.472992975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T19:05:44.243244318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:05:44.250114736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:05:45.165800917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:05:45.169207305Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:05:45.304487688Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T19:05:45.778199352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:05:45.781612569Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:05:45.947098816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" idle=1 full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:05:45.95051876Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T19:05:46.55395413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:49.054547845Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-02-16T19:05:51.554396779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:52.804356566Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T19:05:53.463288557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:05:53.468307541Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:05:53.468331787Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:05:53.468357873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:05:53.471828724Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:05:53.912279121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:05:53.916458829Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:05:54.018930201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:05:54.022624982Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T19:05:54.917597562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40176"
level=info ts=2026-02-16T19:05:56.042752799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:05:56.047754389Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:05:56.534756493Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:05:56.53578695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:05:56.554196493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:05:56.554216963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:05:56.554224867Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:05:56.554239584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:05:56.554261262Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T19:05:56.57552335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:05:56.579136799Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:05:57.469170536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:05:57.474115351Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:06:00.304389183Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-02-16T19:06:01.27918465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41830"
level=warn ts=2026-02-16T19:06:01.554517826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:02.797059225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:06:02.801801664Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:06:03.59532355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:06:03.600078061Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:06:04.018173702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:06:04.022029229Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:06:04.05435196Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-02-16T19:06:04.054380726Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-02-16T19:06:06.553595895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:07.061343388Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:06:07.804615112Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T19:06:09.619167117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:06:10.148901681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:06:10.152650542Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:06:11.554260587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:11.55425401Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T19:06:12.065979583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:06:12.069836856Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:06:12.184378005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:06:12.188189638Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T19:06:13.476016138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T19:06:14.405272025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:06:14.409015068Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:06:14.409035721Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:14.409058399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:06:14.412189069Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:06:15.028243162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" idle=1 full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:06:15.032877927Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:06:15.304531624Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-02-16T19:06:15.533374456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:06:15.537218272Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:06:15.537240723Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:15.537263329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:06:15.540393916Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:06:16.554110956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:19.054364704Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T19:06:21.348295812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:06:21.353807636Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T19:06:21.554532616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:21.916529601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:06:21.921162352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:06:22.804933336Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
ts=2026-02-16T19:06:24.919023993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36004"
level=info ts=2026-02-16T19:06:25.94070013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:06:25.946097686Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:06:26.450591732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:06:26.45435552Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:06:26.553608122Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T19:06:26.553624121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:06:26.553639199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:06:26.553649059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:06:26.554700628Z caller=checkpoint.go:390 msg="flushing series" totalSize="284 kB" series=5
level=info ts=2026-02-16T19:06:26.560646027Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-02-16T19:06:26.563022191Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009185058s
level=info ts=2026-02-16T19:06:28.006530067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:06:28.012027864Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:06:29.02018631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:06:29.023748438Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:06:31.109887402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:06:31.113624405Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:06:31.13996389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:06:31.143699333Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T19:06:31.554024736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:31.78444231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:06:31.787571218Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T19:06:32.800752966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:06:32.807348715Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:06:35.9095346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:06:35.913137132Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:06:35.913159282Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:35.913184013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:06:35.916198461Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T19:06:36.554281173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:37.064104612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:06:37.06790232Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:06:38.588687337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:06:38.592300939Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:06:38.964519239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:06:38.968182224Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:06:39.067432504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:06:39.07104403Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T19:06:39.496311201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T19:06:41.55407079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T19:06:43.477831794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:06:43.732523693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:06:43.736189979Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:06:45.866420491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:06:45.869601698Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T19:06:46.554522454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:48.151742532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:06:48.155417451Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:06:49.616765858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:06:49.622264287Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:06:49.815887759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:06:49.820619339Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:06:51.12945214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:06:51.133754339Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T19:06:51.554378539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:52.893908505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:06:52.897192697Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:06:52.897212076Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:06:52.897233607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:06:52.900665435Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T19:06:53.421630822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56494"
level=info ts=2026-02-16T19:06:53.52844587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:06:53.532383829Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:06:53.785145472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:06:53.788704612Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T19:06:56.253321659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T19:06:56.534848943Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:06:56.535894211Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:06:56.554266718Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T19:06:56.554299969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:06:56.554313751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:06:56.554332211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:06:56.554340469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T19:06:56.554397578Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=debug ts=2026-02-16T19:06:56.5547989Z caller=checkpoint.go:378 msg="writing series" size="481 kB"
level=info ts=2026-02-16T19:06:58.922756691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:06:58.926397477Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:07:00.310628837Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-02-16T19:07:00.310651088Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=2
level=warn ts=2026-02-16T19:07:01.554023389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:04.0556288Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T19:07:05.89063655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:07:05.894530275Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T19:07:06.553620989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:07.805241461Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-02-16T19:07:08.34197145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:07:08.345941056Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:07:08.516359688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:07:08.520246477Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:07:09.189558471Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:07:09.49431885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:09.498463026Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:09.498500713Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:09.498525673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:09.501544922Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:07:10.289988918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:07:10.294258403Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:07:10.416608371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:07:10.420204193Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:07:10.785876511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:07:10.789271893Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:07:11.554543805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:11.55570398Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=info ts=2026-02-16T19:07:12.350865618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:07:12.354113552Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T19:07:13.479612619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T19:07:15.305351846Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:07:15.305375219Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-02-16T19:07:15.947113267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:07:15.950494327Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:07:16.469048488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:07:16.474052951Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T19:07:16.554169163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:19.055431408Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-02-16T19:07:20.199068983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:07:20.20309766Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:07:20.517900573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:07:20.526131448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:07:21.132141563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:07:21.13714463Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:07:21.137166517Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:21.137190266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:07:21.141566831Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T19:07:21.554505255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:21.841690701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:07:21.845976383Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T19:07:22.253670966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:22.257450767Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:07:22.80561147Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
level=info ts=2026-02-16T19:07:22.868746825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:07:22.872581972Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:07:24.565214871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:07:24.56872294Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:07:25.3097424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:07:25.313401762Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T19:07:26.110733123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:26.113909945Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:26.113931028Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:26.11395268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:26.116843321Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:07:26.554452569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:07:26.554496145Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T19:07:26.554471143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:26.55450457Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:07:26.555627336Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T19:07:27.575595535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:07:27.580837404Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:07:27.580859245Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:27.580880418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:07:27.583706511Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T19:07:30.306625233Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-02-16T19:07:30.306651797Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=4
level=info ts=2026-02-16T19:07:30.81618491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:07:30.821983658Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:07:30.822003984Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:30.822028218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:07:30.826628941Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-02-16T19:07:31.288389897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37824"
level=warn ts=2026-02-16T19:07:31.554162338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:31.597050007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:07:31.60123048Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:07:34.055217732Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T19:07:34.745438161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:07:34.748708498Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:07:35.707841989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:07:35.951673411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:07:35.956416364Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T19:07:36.553846432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:37.805712992Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=debug ts=2026-02-16T19:07:37.846517697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:07:38.68511854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:07:38.688394491Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:07:38.758803804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:07:38.761906933Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:07:39.100795622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:07:39.104032521Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:07:41.554541902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:41.554596821Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
ts=2026-02-16T19:07:43.481161807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:07:44.086699303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:07:44.090513146Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T19:07:44.38340386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:07:44.387175266Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:07:45.306094865Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:07:45.306118993Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-02-16T19:07:45.338251441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:45.342424741Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:45.342449357Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:45.342493515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:45.346421825Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:07:46.066792018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:46.072527327Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:46.072548772Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:46.072573564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:07:46.077666462Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T19:07:46.554421905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:48.404766528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:07:48.409677365Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T19:07:49.054795189Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T19:07:49.730981443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:07:49.734296663Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T19:07:51.554039809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:07:52.805233792Z caller=checkpoint.go:378 msg="writing series" size="444 kB"
ts=2026-02-16T19:07:53.425965716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53466"
level=info ts=2026-02-16T19:07:55.19052561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:07:55.193942756Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T19:07:56.535015388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T19:07:56.535013364Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=warn ts=2026-02-16T19:07:56.554385273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:07:56.554390045Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:07:56.554414515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:07:56.554423979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:07:56.55553867Z caller=checkpoint.go:378 msg="writing series" size="7.3 kB"
level=info ts=2026-02-16T19:07:57.155601308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.1 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="12 kB" idle=1 full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:07:57.159546304Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:07:57.880306469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:07:57.887686557Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T19:07:59.222619641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:07:59.226645204Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:07:59.791303204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:07:59.795644096Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:07:59.79566457Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:07:59.795687117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:07:59.798803284Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:07:59.809905594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:07:59.814211767Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T19:08:00.309052578Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T19:08:00.309077019Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.5 MB" series=4
level=info ts=2026-02-16T19:08:01.392389166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:01.397575259Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T19:08:01.553779478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:02.33923592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:08:02.344610233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:08:02.347292871Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:08:02.350274624Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:08:02.81351651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:08:02.817179913Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:08:04.054925549Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=info ts=2026-02-16T19:08:04.634873902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:08:04.638442734Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:08:05.283224509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:08:05.287025416Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:08:05.287048547Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:05.287071329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:08:05.289982242Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:08:05.604871022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:08:05.609893947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:08:05.609925175Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:05.609958584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:08:05.614309353Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:08:06.401634009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:08:06.405771226Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T19:08:06.554278675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:06.803442656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:08:06.808319701Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:08:07.732520808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:07.735976878Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:07.735997105Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:07.736020748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:07.738841405Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T19:08:07.805106164Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=debug ts=2026-02-16T19:08:08.121855148Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:08:08.923990834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:08:08.929449446Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:08:10.640916561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:08:10.644651537Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=warn ts=2026-02-16T19:08:11.554539742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:11.554635304Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T19:08:11.855768744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:08:11.859187617Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T19:08:13.483055591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T19:08:13.738581918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:08:13.742411327Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T19:08:15.304877509Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T19:08:16.5545174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:17.069832896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="725 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:08:17.073790993Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T19:08:19.054679752Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T19:08:19.066767559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:19.070117638Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:08:19.103193657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:08:19.107453203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T19:08:21.554089614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:22.501267413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:08:22.504757222Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:08:22.804893257Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-02-16T19:08:24.361521959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:08:24.365190359Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:08:26.382815907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:08:26.38683464Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T19:08:26.553591162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:26.553594607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:08:26.553621214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:08:26.553629894Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:08:26.555599142Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T19:08:27.198085161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:08:27.203714913Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T19:08:29.275148012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:08:29.27853164Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:08:30.305652967Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=warn ts=2026-02-16T19:08:31.553930669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:31.804645124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:08:31.808404378Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:08:33.832384377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:08:33.835993191Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:08:34.055799417Z caller=checkpoint.go:378 msg="writing series" size="141 kB"
level=warn ts=2026-02-16T19:08:36.553636036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:36.769963929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T19:08:37.825684486Z caller=checkpoint.go:378 msg="writing series" size="29 MB"
level=debug ts=2026-02-16T19:08:37.825711122Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 MB" series=10
level=debug ts=2026-02-16T19:08:38.071382316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T19:08:41.553928577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:08:41.5550958Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-02-16T19:08:42.923798577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:08:42.92723968Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T19:08:43.42962716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:08:43.433217432Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T19:08:43.485588397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=info ts=2026-02-16T19:08:44.676932786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="387 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:08:44.680504188Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T19:08:44.889154896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:08:44.892952391Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:08:45.079458282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:08:45.086909855Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:08:45.305528761Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-02-16T19:08:45.382924733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:08:45.38688725Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T19:08:46.553878106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:48.82519751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:08:48.828807669Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:08:48.828826327Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:48.828858508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:08:48.831725372Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:08:49.056777551Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T19:08:49.056800947Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-02-16T19:08:50.327024039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:08:50.33153621Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T19:08:50.50581156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:08:50.511797903Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:08:50.725628146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:08:50.729055441Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:08:51.554277192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:51.935004214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:08:51.938586633Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:08:52.099047928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:08:52.104052917Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T19:08:52.805570901Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
ts=2026-02-16T19:08:53.430589408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44812"
level=info ts=2026-02-16T19:08:53.600693921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:08:53.604007164Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c679dd7d5%3A19c679ea017%3Ad19b994f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T19:08:53.60402657Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:53.604048833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="21 kB" avg_uncomp="21 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:08:53.606844152Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c679dd7d5%3A19c67ce7e94%3A88b4d4a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T19:08:54.930105985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33500"
level=info ts=2026-02-16T19:08:55.751350207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:08:55.755017274Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:08:56.300264299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.1 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:08:56.303829033Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:08:56.303853815Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:56.303877125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.1 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:08:56.306720031Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:08:56.534159491Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:08:56.535206442Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T19:08:56.553606368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:08:56.553625404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:08:56.553647114Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:08:56.553657383Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T19:08:56.555171007Z caller=checkpoint.go:378 msg="writing series" size="487 kB"
level=info ts=2026-02-16T19:08:56.744405403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="279 kB" avg_comp="140 kB" total_uncomp="2.1 MB" avg_uncomp="1.1 MB" full=2 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T19:08:56.747739498Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c679e9383%3A19c679e9384%3A4fc04fc8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T19:08:56.747759399Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:08:56.74778097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:08:56.751544673Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c679db8e7%3A19c67d4dd64%3Ae9b91026?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-02-16T19:08:57.864045484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:08:57.867640803Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:09:00.09382993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:09:00.098086985Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:09:00.304695087Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T19:09:01.554414875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:04.054783126Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=info ts=2026-02-16T19:09:05.047765114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:09:05.051464748Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T19:09:06.553589733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:07.082342854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:09:07.086527003Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c679dd7d5%3A19c67ce7e94%3A88b4d4a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T19:09:07.256405887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:09:07.259922694Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:09:07.75459327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:09:07.805661078Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-02-16T19:09:08.399408921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:09:08.404740203Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:09:09.736590558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:09:09.740131744Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T19:09:11.553888742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:11.555039086Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=info ts=2026-02-16T19:09:12.40176482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.1 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:09:12.405403043Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:09:13.266614454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:09:13.271938812Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T19:09:13.486781472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T19:09:15.3049828Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=info ts=2026-02-16T19:09:15.578973058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:09:15.582911076Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c679db8e7%3A19c67d4dd64%3Ae9b91026?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-02-16T19:09:16.553634883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:18.637415132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:09:18.640930956Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:09:19.055197398Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T19:09:21.554431829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:21.581592214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:09:21.585396181Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T19:09:22.805285564Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
ts=2026-02-16T19:09:23.431958486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43604"
level=info ts=2026-02-16T19:09:25.277747888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:09:25.281844001Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:09:26.553759642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:09:26.553790183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:09:26.553800938Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:09:26.553795682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:26.554973955Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=debug ts=2026-02-16T19:09:26.554990997Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-02-16T19:09:29.167326249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:09:29.171281487Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T19:09:29.465312919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:09:29.467149678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T19:09:30.215770728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:09:30.221454455Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T19:09:30.304745006Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T19:09:30.422144517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:09:30.426021854Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T19:09:31.301441531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43862"
level=warn ts=2026-02-16T19:09:31.554348582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:32.346233239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:09:32.422281427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:09:32.426569062Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:09:32.426589128Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:09:32.426609135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:09:32.430068653Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T19:09:32.741224762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:09:32.744738496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:09:33.21801819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:09:33.221514945Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:09:34.055148937Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T19:09:34.391253849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:09:34.396059103Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T19:09:35.046268182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:09:35.049754058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c679dd7d5%3A19c67ce7e94%3A88b4d4a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T19:09:35.075827358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:09:35.079097791Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T19:09:36.554285581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:36.824135377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.1 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:09:36.82779272Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T19:09:37.805240417Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T19:09:38.957837866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:09:38.972797017Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T19:09:39.466187345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:09:39.469796939Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T19:09:41.081139653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:09:41.08561344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-02-16T19:09:41.554061337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:41.555321627Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-02-16T19:09:41.798676442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:09:41.803922589Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:09:42.845929409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:09:42.849773075Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:09:43.11145733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:09:43.116773643Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T19:09:43.488076968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T19:09:45.30468752Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T19:09:46.554255108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:49.055223338Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=info ts=2026-02-16T19:09:50.420752795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:09:50.425702232Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c679db8e7%3A19c67d4dd64%3Ae9b91026?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-02-16T19:09:51.290298826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:09:51.293841863Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T19:09:51.554516624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:09:52.132946463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:09:52.136569792Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:09:52.805365668Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T19:09:54.632075117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:09:54.635928116Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T19:09:54.748396987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:09:54.753732758Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:09:55.798783258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="529 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:09:55.802215303Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T19:09:56.534440804Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-02-16T19:09:56.535500008Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T19:09:56.553797389Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:09:56.553825931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:09:56.553836703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:09:56.553865656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:09:56.554964705Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-02-16T19:10:00.305309334Z caller=checkpoint.go:378 msg="writing series" size="639 kB"
level=debug ts=2026-02-16T19:10:00.305334801Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-02-16T19:10:00.671985866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:10:00.677701836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T19:10:01.304309122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37452"
level=warn ts=2026-02-16T19:10:01.553975154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:01.795126446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T19:10:01.798619914Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c679dd7d5%3A19c679e9385%3Aa5016b8e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T19:10:02.239157092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:10:02.24267935Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:10:02.242701518Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:02.242725886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:10:02.245505018Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:10:03.271567402Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T19:10:04.055114045Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
level=warn ts=2026-02-16T19:10:06.553685111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:07.804979533Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-02-16T19:10:10.085813041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.6 kB" total_uncomp="50 kB" avg_uncomp="17 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T19:10:10.089892118Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c679dba98%3A19c67a4ffdc%3A3ab140f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T19:10:11.553996654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:11.555407588Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T19:10:11.555425742Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=3
ts=2026-02-16T19:10:13.490693456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c9ed856e 10.130.0.61:7946"
level=info ts=2026-02-16T19:10:14.412027308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T19:10:14.418129308Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c679e60cc%3A19c679e60cd%3Ae5564d83?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T19:10:15.04538489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="64 kB" avg_comp="64 kB" total_uncomp="394 kB" avg_uncomp="394 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T19:10:15.049463046Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c679db522%3A19c67a0d18d%3A90190138?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T19:10:15.305241936Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=warn ts=2026-02-16T19:10:16.553633348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:17.495345764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=3 total_comp="857 B" avg_comp="285 B" total_uncomp="732 B" avg_uncomp="244 B" full=3 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T19:10:17.499406196Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c679e60cc%3A19c679e9f15%3A22c22425?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T19:10:19.055035926Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=debug ts=2026-02-16T19:10:19.124913241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T19:10:20.553744325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=5 total_comp="438 kB" avg_comp="88 kB" total_uncomp="4.4 MB" avg_uncomp="875 kB" full=5 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T19:10:20.558252736Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c679dafa3%3A19c67aa5edc%3A37f4bcf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T19:10:21.553612723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:22.027390465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:10:22.030908883Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:10:22.807061539Z caller=checkpoint.go:378 msg="writing series" size="3.0 MB"
level=debug ts=2026-02-16T19:10:22.807086925Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 MB" series=3
level=info ts=2026-02-16T19:10:23.220355966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.1 kB" avg_comp="5.1 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T19:10:23.223782526Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c679dd7d5%3A19c67ce7e94%3A88b4d4a0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T19:10:23.436278813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39680"
level=info ts=2026-02-16T19:10:24.502521628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="649 kB" avg_comp="649 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-02-16T19:10:24.508079752Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19c679dd7d5%3A19c67b16f74%3A1b2891d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-02-16T19:10:24.698800554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="667 kB" avg_comp="667 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T19:10:24.703045364Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c679dd7d5%3A19c67b55f45%3A986d376a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T19:10:24.728345932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="18 kB" avg_comp="8.9 kB" total_uncomp="100 kB" avg_uncomp="50 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T19:10:24.732127442Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c679e60cc%3A19c67a4ffdc%3Ad0b9dcf8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T19:10:25.943605183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:25.94821959Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:25.9482395Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:25.948261063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:25.952804986Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:10:26.55350557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T19:10:26.553538031Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T19:10:26.553554621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T19:10:26.553571405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:26.555219589Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-02-16T19:10:28.470598951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.1 kB" avg_comp="2.0 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T19:10:28.475628063Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c679e9383%3A19c67a0e2cc%3Ae21d0423?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T19:10:28.607155161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T19:10:28.610894368Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c679dc0b3%3A19c679e9384%3A88586519?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T19:10:29.218446522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=6 total_comp="2.3 kB" avg_comp="390 B" total_uncomp="2.0 kB" avg_uncomp="341 B" full=6 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T19:10:29.222235036Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c67a5750b%3A19c67a5750c%3Af56bbdfb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T19:10:30.305444947Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T19:10:31.076770047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-02-16T19:10:31.081159732Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19c679db8e7%3A19c67d4dd64%3Ae9b91026?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-02-16T19:10:31.307169406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57748"
level=warn ts=2026-02-16T19:10:31.554440828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:32.23889931Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T19:10:32.507722679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="303 kB" avg_uncomp="101 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T19:10:32.511640271Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c679dc0fe%3A19c67c5c621%3Af32c1c04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T19:10:32.871831244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7a2bc6e82dd50e48 immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-qfzn\"}"
level=error ts=2026-02-16T19:10:32.875829167Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7a2bc6e82dd50e48/19c67a551e3%3A19c67aebdac%3Ae195eac8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qfzn\"}"
level=info ts=2026-02-16T19:10:33.121371836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:33.128182143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:33.128203974Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:33.128231146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:33.135276045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T19:10:34.055539352Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=info ts=2026-02-16T19:10:35.719943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="8.8 MB" avg_uncomp="8.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T19:10:35.72465086Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c679db7ac%3A19c67c1e40e%3Ac6774fc0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T19:10:36.510129494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T19:10:36.51337266Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c679e9f14%3A19c679eeccc%3A9e186e2e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T19:10:36.55368209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:37.01086611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="996 kB" avg_comp="996 kB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T19:10:37.015232218Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c679dbe88%3A19c67d5b43c%3Af829190f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T19:10:37.136528074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:37.139936919Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:37.139961215Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T19:10:37.139992003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T19:10:37.143036539Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c67a5a7d3%3A19c67a77c34%3A5ca8d764?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T19:10:37.805034794Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T19:10:41.553919842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T19:10:41.555017932Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T19:10:42.055283096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T19:10:42.05962273Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c679dd7d5%3A19c679e9f14%3A208dd264?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T19:10:43.034920171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="734 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T19:10:43.03824653Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c679dd7d5%3A19c679e9f15%3A411dc5bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T19:10:43.492662832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T19:10:43.718925251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.6 MB" avg_uncomp="7.6 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T19:10:43.724593495Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c679db0eb%3A19c67a941b5%3Abbaa697c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T19:10:45.305159525Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T19:10:45.607422571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.2 MB" avg_comp="2.3 MB" total_uncomp="81 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T19:10:45.613954363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c679dc19b%3A19c679f8028%3A54cf40f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T19:10:46.553703985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T19:10:47.06989804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="538 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T19:10:47.073576868Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c679dd7d5%3A19c679dd7d6%3A4124919f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T19:10:48.087721942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T19:10:48.091024708Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c67a0e8fb%3A19c67a0e8fc%3A97b26e3d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T19:10:49.055180878Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"