level=info ts=2026-02-16T13:54:01.737704349Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T13:54:01.737733472Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T13:54:01.73808773Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T13:54:01.73883866Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T13:54:01.739216593Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:54:01.739304525Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.59 inf=eth0
level=info ts=2026-02-16T13:54:01.74085782Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-4ef36c79
ts=2026-02-16T13:54:01.740922204Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T13:54:01.74093614Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.59 advertisePort=7946
level=debug ts=2026-02-16T13:54:01.740952706Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.59 advertisePort=7946
level=info ts=2026-02-16T13:54:01.741109062Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T13:54:01.745616885Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T13:54:01.745651625Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-02-16T13:54:01.747257481Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T13:54:01.747847298Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T13:54:01.747865131Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T13:54:01.747863072Z
level=debug ts=2026-02-16T13:54:01.747883576Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T13:54:01.747905278Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T13:54:01.747930309Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T13:54:01.74793842Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T13:54:01.747936938Z
level=debug ts=2026-02-16T13:54:01.747945846Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T13:54:01.74798443Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-02-16T13:54:01.749835432Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T13:54:01.74986364Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:54:01.749878778Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.771887ms
level=error ts=2026-02-16T13:54:01.749890456Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T13:54:01.749908648Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T13:54:01.751243086Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:54:01.751901509Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.59 inf=eth0
ts=2026-02-16T13:54:01.754117864Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:01.754135485Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:54:01.766820476Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-02-16T13:54:01.766831216Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-02-16T13:54:01.76684771Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.766859964Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-02-16T13:54:01.766869206Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T13:54:01.766885406Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T13:54:01.766885933Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-02-16T13:54:01.76692722Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T13:54:01.766926733Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:54:01.76716896Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-02-16T13:54:01.767181986Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T13:54:01.767218182Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-02-16T13:54:01.767232188Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=info ts=2026-02-16T13:54:01.767243942Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T13:54:01.767247724Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-02-16T13:54:01.767259208Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T13:54:01.767287102Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.767291589Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-02-16T13:54:01.767295234Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T13:54:01.76738032Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:54:01.767387362Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-02-16T13:54:01.767390924Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-02-16T13:54:01.76739425Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-02-16T13:54:01.767397226Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-02-16T13:54:01.767423382Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T13:54:01.767445485Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-02-16T13:54:01.767478856Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-02-16T13:54:01.767543984Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=101.068µs errors=false
level=info ts=2026-02-16T13:54:01.767554103Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-02-16T13:54:01.76770878Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=266.322µs errors=false
level=info ts=2026-02-16T13:54:01.767720389Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T13:54:01.767753034Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=310.139µs
level=info ts=2026-02-16T13:54:01.767834561Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T13:54:01.768268524Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T13:54:01.768291472Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-02-16T13:54:01.768297086Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=3.156488296s
level=info ts=2026-02-16T13:54:01.768376868Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=debug ts=2026-02-16T13:54:01.768383984Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T13:54:01.768390229Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-02-16T13:54:01.768387862Z caller=loki.go:599 msg="Loki started" startup_time=96.597301ms
level=debug ts=2026-02-16T13:54:01.76846607Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-02-16T13:54:02.768570968Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:02.768601072Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:06.684790324Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:06.684816501Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:54:06.768496495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:54:11.768535795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:54:11.945446907Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:54:11.945471485Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:54:16.769193621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:54:17.842656863Z caller=logging.go:137 msg="GET /ready (503) 131.216µs"
level=warn ts=2026-02-16T13:54:21.76870243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:54:22.010426328Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.61:7946: dial tcp 10.129.0.61:7946: connect: connection refused"
ts=2026-02-16T13:54:22.01129233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.60:7946"
ts=2026-02-16T13:54:22.012596245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.58:7946"
ts=2026-02-16T13:54:22.014423916Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
level=info ts=2026-02-16T13:54:22.014437625Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.264521869s
level=warn ts=2026-02-16T13:54:26.76946556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:54:27.840539883Z caller=logging.go:137 msg="GET /ready (503) 76.796µs"
level=info ts=2026-02-16T13:54:31.768926877Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T13:54:31.768960753Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T13:54:31.768968519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:54:31.76900503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:54:33.141314073Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T13:54:36.769582584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:54:41.769158793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:54:46.768698409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:54:51.769369025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:54:55.795247136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:43282"
level=warn ts=2026-02-16T13:54:56.768895568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:54:56.971190393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:57.234972131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T13:54:59.091267811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T13:55:01.748085845Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T13:55:01.748146702Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T13:55:01.76913333Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T13:55:01.769145654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:55:01.769168597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:55:01.769178868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:55:06.769110546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:55:11.768675284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:55:16.768895845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:55:17.992549739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:55:21.768922378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:55:23.766034105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:37478"
level=debug ts=2026-02-16T13:55:25.39203078Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T13:55:26.768835283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:55:27.236927987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T13:55:31.768717416Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:55:31.768754377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:55:31.76876584Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:55:31.768811985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:55:36.769512945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:55:40.731555283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:55:41.768985488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:55:46.769030613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:55:51.768638853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:55:56.769113057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:55:57.238730228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T13:56:01.335666977Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T13:56:01.747943981Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T13:56:01.749015371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T13:56:01.769456999Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:56:01.769487695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:56:01.769495719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:56:01.769487885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:56:05.124953807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:56:06.769259893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:56:11.769061854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:56:16.768458516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:56:21.042209612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44138"
level=warn ts=2026-02-16T13:56:21.768899544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:56:25.798877912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:36984"
level=warn ts=2026-02-16T13:56:26.768893953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:56:27.240717982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T13:56:29.837389366Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T13:56:31.768426691Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:56:31.768458223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:56:31.768466334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:56:31.768459926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:56:36.769014838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:56:38.796433423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:56:38.798059351Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:56:41.769339216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:56:46.769274349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:56:51.044723179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50708"
level=warn ts=2026-02-16T13:56:51.768668171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:56:53.770882505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55648"
ts=2026-02-16T13:56:55.800022114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46084"
level=debug ts=2026-02-16T13:56:55.974919689Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T13:56:56.769349439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:56:57.242278327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T13:57:01.748765144Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T13:57:01.749547559Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T13:57:01.768910198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:57:01.768939106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:57:01.768948482Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:57:01.768947395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:06.769304425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:11.769199855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:16.768997193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:21.76860015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:57:23.050994409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:57:24.726205333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T13:57:26.769311403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:57:27.24385057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T13:57:31.768809623Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:57:31.768839872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:57:31.768847768Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:57:31.768874402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:36.768842389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:41.768744134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:46.768982206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:57:51.769320556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:57:51.835625681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T13:57:51.835691694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T13:57:51.847053097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T13:57:55.803484181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:36042"
level=warn ts=2026-02-16T13:57:56.769197941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:57:57.245715079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T13:58:01.74803638Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T13:58:01.748048017Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T13:58:01.768520202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T13:58:01.768534821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:58:01.768551804Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:58:01.768559859Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T13:58:04.672183987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T13:58:04.680826548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T13:58:06.769511938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:58:11.769017398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:58:16.769190317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:58:21.03535453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:58:21.769185649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:58:24.071726216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T13:58:26.768848365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:58:27.247090817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T13:58:31.769080569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:58:31.769110329Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:58:31.769118479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:58:31.769079828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T13:58:32.552126082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T13:58:32.561354293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T13:58:36.769294326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:58:41.768502653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:58:46.768656654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:58:50.215616798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T13:58:51.053233525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52578"
level=warn ts=2026-02-16T13:58:51.768828202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T13:58:56.769123477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:58:57.249011926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T13:59:01.7479843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T13:59:01.748067373Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T13:59:01.768104894Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T13:59:01.76835591Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-02-16T13:59:01.768440857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:59:01.768465484Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:59:01.768473977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:59:01.768514134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:01.768628464Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-02-16T13:59:06.269180119Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-02-16T13:59:06.768864408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:10.768707595Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=warn ts=2026-02-16T13:59:11.768769594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:12.247253555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:59:15.26920058Z caller=checkpoint.go:378 msg="writing series" size="5.6 kB"
level=debug ts=2026-02-16T13:59:16.56462341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T13:59:16.769082246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:19.769089452Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=warn ts=2026-02-16T13:59:21.768654271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:59:23.779821306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:51912"
level=debug ts=2026-02-16T13:59:24.269010265Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=info ts=2026-02-16T13:59:24.902525327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T13:59:24.911735748Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T13:59:25.808928163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:55314"
level=warn ts=2026-02-16T13:59:26.768483332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:59:27.250112374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T13:59:28.769188498Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-02-16T13:59:31.768776027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T13:59:31.768804238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T13:59:31.768813268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T13:59:31.768818124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:33.269234232Z caller=checkpoint.go:378 msg="writing series" size="268 kB"
level=warn ts=2026-02-16T13:59:36.769377887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:37.769189098Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-02-16T13:59:41.768661004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:42.269510758Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=debug ts=2026-02-16T13:59:42.269535458Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-02-16T13:59:46.768912493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:46.768938865Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=info ts=2026-02-16T13:59:47.240727386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="219 B" avg_comp="219 B" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" idle=1 labels="{stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T13:59:47.240761356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T13:59:47.245647441Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T13:59:47.541725387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="44 kB" avg_uncomp="44 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T13:59:47.545907485Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T13:59:47.840750347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="132 B" avg_comp="132 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T13:59:47.844556621Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T13:59:48.14146265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="8.1 kB" avg_comp="8.1 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T13:59:48.145307548Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T13:59:48.441175008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="5.0 kB" avg_comp="5.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T13:59:48.445048956Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T13:59:48.7407737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T13:59:48.744352323Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T13:59:49.041378811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T13:59:49.04693859Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T13:59:49.341117558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="54 kB" avg_uncomp="54 kB" idle=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T13:59:49.34534035Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T13:59:49.641389914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="44 kB" avg_uncomp="44 kB" idle=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T13:59:49.644979686Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T13:59:49.941487964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T13:59:49.945086988Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T13:59:50.241628116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T13:59:50.245243671Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T13:59:51.269248268Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-02-16T13:59:51.769058953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T13:59:55.770491649Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-02-16T13:59:56.768934377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T13:59:57.251453665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T13:59:58.950027931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T13:59:58.953858744Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T13:59:59.90255915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T13:59:59.906032162Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:00:00.268925193Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
level=info ts=2026-02-16T14:00:00.847776229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:00:00.851352259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:00:01.587788493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:00:01.59117612Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:00:01.703558669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:00:01.707057944Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:00:01.747963826Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:00:01.749031379Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:00:01.749273601Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T14:00:01.74931957Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968055 n=1
level=debug ts=2026-02-16T14:00:01.749364929Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968055 n=1
level=info ts=2026-02-16T14:00:01.768386929Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:00:01.768448734Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:00:01.768456903Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:00:01.768448883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:02.981498363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:00:02.984896479Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:00:04.769479954Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T14:00:05.976195662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:00:05.979532341Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:00:06.139595708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:00:06.144373315Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:00:06.507451159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:00:06.510805783Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:00:06.768672525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:07.011051884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:00:07.01491086Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:00:07.813449136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:00:07.817931561Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:00:09.268642138Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-02-16T14:00:11.597078137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:00:11.59708196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:00:11.600549439Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:00:11.768733875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:11.89808123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="7.4 kB" avg_comp="7.4 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" idle=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:00:11.901520042Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:00:12.19754741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:00:12.200904218Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:00:12.497664463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:00:12.500916422Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:00:12.750816162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:00:12.758532113Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:00:12.797625494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="16 kB" avg_uncomp="16 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:00:12.800999086Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:00:13.768810405Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-02-16T14:00:15.344137011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T14:00:16.768736082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:00:18.269050474Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-02-16T14:00:21.060134037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:51218"
level=warn ts=2026-02-16T14:00:21.769450228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:21.932808246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:00:21.936273152Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:00:22.77071259Z caller=checkpoint.go:378 msg="writing series" size="3.8 MB"
level=debug ts=2026-02-16T14:00:22.770738667Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.2 MB" series=9
level=info ts=2026-02-16T14:00:23.640460621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:00:23.643913037Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T14:00:23.782568566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:42918"
ts=2026-02-16T14:00:25.81264015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:41440"
level=warn ts=2026-02-16T14:00:26.769288287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:00:27.252610508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:00:27.269658763Z caller=checkpoint.go:378 msg="writing series" size="500 kB"
level=info ts=2026-02-16T14:00:27.540203114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:00:27.544394445Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:00:28.163903013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:00:28.167469185Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:00:29.491261032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:00:29.494594439Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:00:31.217629597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:00:31.221223585Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:00:31.369734534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:00:31.373119055Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:00:31.76908983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:31.769116002Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:00:31.769119855Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T14:00:31.769136521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:00:31.769144898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:00:32.316271319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:00:32.319671534Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:00:34.198667487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:00:34.202014967Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:00:34.92838274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:00:34.931681452Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T14:00:35.397953897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:00:35.401381288Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:00:36.269107121Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
level=warn ts=2026-02-16T14:00:36.7691754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:37.866856676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:00:37.870347571Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:00:39.353109779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:00:39.516354525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:00:39.519770011Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:00:40.111415552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:00:40.114955092Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:00:40.769226918Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-02-16T14:00:41.768902103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:00:42.852816205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:00:42.856371531Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:00:45.225460081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:00:45.228591924Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:00:45.268888187Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=info ts=2026-02-16T14:00:45.839200313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:00:45.842683572Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:00:46.768692225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:00:49.769710026Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-02-16T14:00:51.062197386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60530"
level=info ts=2026-02-16T14:00:51.170017384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:00:51.173732366Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:00:51.408353426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:00:51.412061991Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:00:51.768685958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:00:54.269194391Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-02-16T14:00:54.607206342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:00:54.611001468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T14:00:56.769198357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:00:57.254253205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:00:58.769458127Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=debug ts=2026-02-16T14:00:58.769482779Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-02-16T14:01:01.747665318Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:01:01.748743634Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:01:01.769121725Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:01:01.769146205Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:01:01.769154016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:01:01.769170221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:01:03.269228497Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T14:01:03.34765475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:01:03.3516889Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:01:05.030363775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:01:05.037800151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:01:06.054635955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:01:06.058183336Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:01:06.768908612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:01:07.768674156Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=debug ts=2026-02-16T14:01:08.024381628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:01:10.849849436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:01:10.854324118Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:01:11.769227682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:01:12.152712342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:01:12.156328532Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:01:12.268856519Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T14:01:16.768755585Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-02-16T14:01:16.768816271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:01:16.889114119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:01:16.893081697Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T14:01:21.064584159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38688"
level=debug ts=2026-02-16T14:01:21.269502746Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T14:01:21.768918691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:01:23.785505958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:42646"
level=info ts=2026-02-16T14:01:24.454686763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:01:24.45820928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:01:24.563674425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:01:24.566961352Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:01:25.271114884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:01:25.274776996Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:01:25.390182652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:01:25.393819687Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:01:25.769068454Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
ts=2026-02-16T14:01:25.816579402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:39346"
level=warn ts=2026-02-16T14:01:26.769294453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:01:27.187728053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:01:27.191581291Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T14:01:27.256471736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:01:29.62617088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:01:29.629647421Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:01:30.268967983Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=info ts=2026-02-16T14:01:31.7687345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:01:31.76876693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:01:31.768778353Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:01:31.768773672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:01:32.400748332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:01:32.404366932Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:01:32.97770703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:01:32.979630991Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:01:34.769384879Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=debug ts=2026-02-16T14:01:36.208216155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:01:36.76926641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:01:38.103535388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:01:38.107532985Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:01:39.269121046Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=info ts=2026-02-16T14:01:40.301028775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:01:40.30635853Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:01:41.769341439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:01:43.769161259Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T14:01:44.824599579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:01:44.82836651Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:01:44.850596134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:01:44.854085846Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T14:01:46.769444298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:01:48.269502468Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T14:01:49.566342089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:01:49.570372538Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:01:51.768533956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:01:52.769457549Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-02-16T14:01:53.924362999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:01:53.927874627Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:01:55.653623589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:01:55.66127087Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:01:56.769470815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:01:57.257823824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:01:57.269566482Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:02:00.207269949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:02:00.212082297Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:02:01.747955108Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:02:01.749029953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:02:01.76838925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:02:01.768433864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:02:01.768442559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:02:01.768453554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:01.769576241Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=info ts=2026-02-16T14:02:01.835770929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:02:01.839882249Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T14:02:05.268618154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:02:05.268686819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="424 kB" avg_uncomp="424 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:02:05.272525327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:02:06.269478754Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-02-16T14:02:06.769012361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:02:09.0688344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:02:09.072891939Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:02:09.18821937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:02:09.191634032Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:02:09.413173789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:02:09.416752246Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:02:10.347982057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:02:10.353394897Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:02:10.769375528Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=warn ts=2026-02-16T14:02:11.768798686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:15.268859251Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T14:02:15.3200649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:02:15.323683487Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:02:16.598139987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:02:16.769357492Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:19.7695642Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-02-16T14:02:20.807141425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:02:20.810645875Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:02:21.60385995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:02:21.607166872Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:02:21.769331525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:02:22.250824175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:02:22.254293738Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:02:23.742914605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:02:23.746354633Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:02:24.167381052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:02:24.171275223Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:02:24.26958077Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-02-16T14:02:25.819803953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:55748"
level=warn ts=2026-02-16T14:02:26.769326586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:02:27.258742135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:02:27.321528238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:02:27.325352609Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:02:27.711059229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:02:27.714935029Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:02:28.769150281Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T14:02:31.352842163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:02:31.357889895Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:02:31.769218288Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:02:31.769216857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:31.769247618Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:02:31.769258917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:02:33.268785096Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=debug ts=2026-02-16T14:02:33.268819134Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=21
level=debug ts=2026-02-16T14:02:36.609388293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:02:36.609464667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:02:36.61318318Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:02:36.769397429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:37.769218395Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=warn ts=2026-02-16T14:02:41.768590066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:42.269211885Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T14:02:42.992474043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:02:42.997489082Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:02:45.188571496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:02:45.192226861Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:02:46.761517406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:02:46.76491773Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:02:46.768984515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:02:46.769577928Z caller=checkpoint.go:378 msg="writing series" size="655 kB"
level=info ts=2026-02-16T14:02:48.059465276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:02:48.063458486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:02:49.97662693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:02:49.981515577Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:02:50.913856411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:02:50.922335432Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:02:51.269623564Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-02-16T14:02:51.769305494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:02:53.078976439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:02:53.082933617Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T14:02:53.789950564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:47924"
level=info ts=2026-02-16T14:02:55.267625808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:02:55.272744684Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T14:02:55.769340741Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
ts=2026-02-16T14:02:55.821179897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:48406"
level=warn ts=2026-02-16T14:02:56.769079568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:02:57.260310646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:03:00.131247794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:03:00.135136052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:03:00.269538197Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:03:01.748045444Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:03:01.748247288Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:03:01.768643493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:03:01.768667848Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:03:01.76867744Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:03:01.7686983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:03:03.728346657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:03:03.73226166Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:03:04.769532469Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=info ts=2026-02-16T14:03:05.760450034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:03:05.764478607Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:03:06.768912117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:03:07.546537615Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:03:07.546604695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="24 kB" avg_uncomp="24 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:03:07.550108547Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:03:07.560176426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:03:07.563466844Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:03:08.062985949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:03:08.066374989Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:03:08.8933376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:03:08.896771055Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:03:09.268946977Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=debug ts=2026-02-16T14:03:09.268971557Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-02-16T14:03:10.456770397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:03:10.460232093Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:03:11.769057365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:03:13.769144158Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T14:03:14.184264936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:03:14.187910519Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:03:15.959133797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:03:16.769469244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:03:16.940014221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:03:16.943866843Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:03:18.269448278Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T14:03:18.64554119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:03:18.649364827Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:03:21.769397686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:03:22.770495895Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T14:03:22.770532886Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-02-16T14:03:25.033052443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:03:25.037205423Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:03:25.534491976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:03:25.538471166Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:03:26.565806256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:03:26.570328081Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:03:26.768716621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:03:27.261548374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:03:27.269339446Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T14:03:30.238361557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:03:30.242613183Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:03:30.428017373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:03:30.432188522Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:03:31.768520857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:03:31.768555371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:03:31.76856891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:03:31.768600717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:03:31.768592841Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 kB" series=1
level=info ts=2026-02-16T14:03:31.770733934Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-02-16T14:03:31.773982767Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005424525s
level=warn ts=2026-02-16T14:03:36.768835012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:03:39.492613873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:03:39.492695584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:03:39.497674696Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:03:39.656373061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:03:39.660450396Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:03:41.769228419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:03:43.202147247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:03:43.206184822Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:03:46.321101954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:03:46.326423295Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:03:46.569238511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:03:46.577219543Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:03:46.768529731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:03:47.291458235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:03:47.295269747Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:03:47.324347597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:03:47.32816499Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T14:03:47.669643691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:03:47.6735143Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T14:03:51.076561424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47200"
level=warn ts=2026-02-16T14:03:51.768777902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:03:51.856103524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:03:51.859975514Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:03:52.379538198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:03:52.383158824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-02-16T14:03:53.793530577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:41244"
level=info ts=2026-02-16T14:03:54.914051859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:03:54.918578116Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T14:03:55.824880104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:55732"
level=info ts=2026-02-16T14:03:56.038906419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:03:56.042879409Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T14:03:56.769346282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:03:57.263026443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:03:58.46820345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:03:58.472805019Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:04:00.068364235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:04:00.072633476Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:04:01.747574662Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:04:01.748639147Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:04:01.76795576Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:04:01.768133972Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T14:04:01.768453706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:04:01.768462449Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=debug ts=2026-02-16T14:04:01.768475097Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T14:04:01.768465322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:04:01.768483712Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:04:01.900363623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:04:01.905314938Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:04:04.307472619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:04:04.464854004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:04:04.46863198Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:04:05.681726015Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=warn ts=2026-02-16T14:04:06.7688412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:04:07.633294696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:04:07.637559397Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:04:09.102289345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:04:09.595215847Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T14:04:09.648472161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:04:09.652031191Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:04:11.709831564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:04:11.713322736Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:04:11.768629409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:04:13.508176372Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=info ts=2026-02-16T14:04:15.947256531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:04:15.952759999Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:04:16.769351535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:04:17.420972629Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:04:20.27920517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:04:20.285892417Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T14:04:21.079239835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43054"
level=debug ts=2026-02-16T14:04:21.334526455Z caller=checkpoint.go:378 msg="writing series" size="333 kB"
level=warn ts=2026-02-16T14:04:21.768887738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:04:22.607780685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:04:22.612734991Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:04:23.336911185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:04:23.341715138Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:04:23.350753308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:04:23.35401483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:04:25.247013218Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=debug ts=2026-02-16T14:04:25.247046854Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-02-16T14:04:26.76915149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:04:27.264983531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:04:29.159746743Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:04:29.767280493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:04:29.772958491Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T14:04:31.769237621Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:04:31.769250215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:04:31.769275803Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:04:31.769286891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:04:32.042147492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:04:32.045994017Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:04:33.073794087Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=debug ts=2026-02-16T14:04:33.878214093Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:04:36.76886475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:04:36.986121335Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T14:04:39.5503252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:04:39.553998359Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:04:40.654096612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:04:40.658006095Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:04:40.900057754Z caller=checkpoint.go:378 msg="writing series" size="656 kB"
level=warn ts=2026-02-16T14:04:41.768999634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:04:43.121717527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:04:43.125838247Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:04:44.812627566Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T14:04:44.916889298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:04:44.923896744Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:04:44.923920828Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:04:44.923947559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:04:44.932347369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:04:46.230218723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:04:46.233816621Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:04:46.366243269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:04:46.370264662Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:04:46.769318894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:04:48.725041554Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T14:04:49.395187996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:04:49.398719175Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:04:49.775766259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:04:49.779158599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:04:50.625341858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:04:50.630935053Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T14:04:51.76892941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:04:52.638891103Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=debug ts=2026-02-16T14:04:56.551242287Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-02-16T14:04:56.768670835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:04:57.267045415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:04:57.88084667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:04:57.889578901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:04:58.019429773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:04:58.022909721Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:04:59.758615761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:04:59.762177568Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:05:00.178526526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:05:00.182474286Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:05:00.464328413Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=debug ts=2026-02-16T14:05:00.464353292Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-02-16T14:05:00.601350288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:05:01.747704418Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:05:01.748762524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T14:05:01.769171586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:01.76917089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:05:01.769202334Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:05:01.769213799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:05:02.971608121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:05:02.975544931Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:05:04.294610844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:05:04.298188742Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:05:04.377984289Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T14:05:06.342035022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:05:06.346189775Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:05:06.718199862Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:05:06.769137958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:08.278427582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:05:08.282012769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:05:08.291066758Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T14:05:11.769418255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:05:12.204030078Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T14:05:12.846069882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:05:12.850108635Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:05:15.692179334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:05:15.696201654Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:05:16.116424914Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-02-16T14:05:16.768487175Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:18.381026243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:05:18.384721203Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:05:20.030094552Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T14:05:20.370286975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:05:20.374748912Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:05:20.725749368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:05:20.730298477Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T14:05:21.083436838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58298"
level=warn ts=2026-02-16T14:05:21.768796719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:22.25221885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:05:22.259171061Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:05:23.945443318Z caller=checkpoint.go:378 msg="writing series" size="4.1 MB"
level=debug ts=2026-02-16T14:05:23.945469482Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.2 MB" series=6
level=info ts=2026-02-16T14:05:24.622866074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:05:24.626337596Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:05:26.7693646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:05:27.268369933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:05:27.314151904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:05:27.318520954Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:05:27.855724545Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T14:05:30.955276772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:05:30.960633438Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:05:31.768663065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:05:31.768700733Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:05:31.768715426Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:05:31.76874882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:05:31.769035746Z caller=checkpoint.go:378 msg="writing series" size="595 kB"
level=debug ts=2026-02-16T14:05:35.681640941Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-02-16T14:05:36.768913409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:05:36.770031173Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:05:38.467232366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:05:38.471315619Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:05:39.594901301Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T14:05:39.769465644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:05:39.773135979Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:05:40.258520008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:05:40.262017684Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:05:41.768688129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:42.053830381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:05:42.057053134Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:05:42.220371041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:05:42.223731636Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:05:42.756298491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:05:42.758057874Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:05:43.50810112Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-02-16T14:05:44.864782817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:05:44.86846995Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T14:05:46.768659159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:47.312608367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:05:47.316435766Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:05:47.421614125Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=debug ts=2026-02-16T14:05:47.421639998Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-02-16T14:05:48.78729689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:05:48.790976447Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:05:48.790997201Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:05:48.791022154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:05:48.794067822Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:05:51.334337455Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-02-16T14:05:51.768909944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:52.17945425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:05:52.183045003Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:05:55.246831965Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T14:05:56.768590307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:05:56.771673698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:05:56.774940761Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:05:56.774958392Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:05:56.774978166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:05:56.777690715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:05:57.269510772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:05:58.152526365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:05:58.157487853Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:05:58.877560849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:05:58.881028839Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:05:59.15991713Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=info ts=2026-02-16T14:05:59.522683165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:05:59.526049062Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:05:59.632473699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:05:59.635851368Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:05:59.635870517Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:05:59.635890704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:05:59.639534674Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:06:01.599869353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:06:01.609589236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:06:01.748048721Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:06:01.748055079Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:06:01.768527795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:06:01.768555802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:06:01.768564197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:06:01.768642098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:06:03.073517184Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-02-16T14:06:06.769074652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:06:06.98588228Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T14:06:09.04513262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:06:09.04999822Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c66bb381e%3A19c66bb381f%3Ab657c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T14:06:09.050022702Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:09.050052563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="27 kB" avg_uncomp="27 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:06:09.053178021Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:06:09.242464122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:06:09.249147962Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:06:09.759595926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:06:09.763326923Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:06:10.899345705Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T14:06:11.445182446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:06:11.449072639Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:06:11.76903741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:06:12.723418376Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:06:12.723504208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="344 B" avg_comp="344 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:06:12.728065331Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:06:14.812342261Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T14:06:14.8905944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:06:14.894316144Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:06:15.176188482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:06:15.180255261Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:06:16.104741713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:06:16.108277253Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:06:16.108305599Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:16.10833741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:06:16.111521294Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T14:06:16.769126893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:06:18.725014738Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-02-16T14:06:18.901823081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:06:18.905435081Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:06:21.769310683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:06:22.447784611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:06:22.452270854Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:06:22.638890682Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:06:23.846294381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:06:23.850241835Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:06:23.850264639Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:23.850288725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:06:23.85345844Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:06:25.831971747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:33090"
level=info ts=2026-02-16T14:06:25.981931979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:06:25.985574637Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:06:26.024640474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:06:26.028452646Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:06:26.028477242Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:26.028503282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:06:26.031882186Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:06:26.551549746Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=warn ts=2026-02-16T14:06:26.769017797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:06:27.271315168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:06:28.195070308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:06:28.199242274Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:06:30.002363957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:06:30.005967522Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:06:30.005991214Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:30.006013647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:06:30.008944131Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:06:30.464228147Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:06:31.665298353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:06:31.67189557Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:06:31.671921843Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:31.671946626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:06:31.674919474Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:06:31.769012111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:06:31.769043361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:06:31.76905173Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:06:31.769078681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:06:32.002107561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:06:32.005630664Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:06:32.788476099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:06:32.793151896Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:06:33.312463608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:06:33.316466541Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:06:33.316495075Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:33.316525244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:06:33.320044054Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:06:34.215166767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:06:34.21880105Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:06:34.218827754Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:34.218853659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:06:34.222044839Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:06:34.377738057Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=info ts=2026-02-16T14:06:34.792462583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:06:34.796213291Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:06:36.082701822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:06:36.086195261Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:06:36.257699804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:06:36.76894001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:06:37.804321224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:06:37.807937912Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:06:37.807958637Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:37.807986278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:06:37.810772545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:06:38.290984725Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=info ts=2026-02-16T14:06:38.425368814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:06:38.428823496Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:06:38.791020159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:06:38.794751006Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T14:06:38.794777525Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:39.095521054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:06:39.098815543Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:06:39.323662313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:06:40.458103984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:06:40.461755796Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:06:40.484853646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:06:40.488131325Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:06:40.488167633Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:40.488200866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:06:40.491443499Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:06:41.769065024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:06:42.20380029Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:06:43.418674841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:06:43.422555299Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:06:45.067873179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:06:45.072032229Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:06:45.20938929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:06:45.213142442Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:06:46.116952015Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T14:06:46.190395664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:06:46.19421247Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:06:46.194236372Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:06:46.194260925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:06:46.197576462Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T14:06:46.769121366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:06:50.029687707Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-02-16T14:06:51.580483412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:06:51.584770789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:06:51.769343424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:06:52.298174003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:06:52.302641095Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:06:52.745811006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:06:52.750145838Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:06:52.994886652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:06:52.99891219Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:06:53.942959154Z caller=checkpoint.go:378 msg="writing series" size="374 kB"
level=debug ts=2026-02-16T14:06:53.942987607Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=17
level=warn ts=2026-02-16T14:06:56.768778973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:06:56.88203449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:06:56.887238642Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T14:06:57.272818355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:06:57.276789542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:06:57.281949426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:06:57.856349088Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-02-16T14:06:58.707821861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:06:58.716780686Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:06:58.942928246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:06:58.9469106Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:07:01.747992915Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:07:01.748033844Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:07:01.768453932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:07:01.768458244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:07:01.768485239Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:07:01.768492902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:07:01.76859057Z caller=checkpoint.go:378 msg="writing series" size="275 kB"
level=info ts=2026-02-16T14:07:03.640002631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:07:03.64527436Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:07:04.680630831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:07:04.684157665Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:07:04.775836455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:07:04.780697237Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:07:04.919919141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:07:04.923992667Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:07:05.362330947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:07:05.366128928Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:07:05.682036224Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:07:06.380252741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:07:06.384259394Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:07:06.384283238Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:07:06.384311169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:07:06.387776324Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:07:06.768718609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:07:08.088229982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:07:08.091943594Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:07:08.516228063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:07:08.51994703Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:07:09.595034625Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-02-16T14:07:10.645483746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:07:10.64943373Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:07:10.78905029Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:07:11.768760728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:07:13.045144822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:07:13.048941985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:07:13.508240645Z caller=checkpoint.go:378 msg="writing series" size="867 B"
level=info ts=2026-02-16T14:07:13.696242022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:07:13.700063957Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:07:15.771112285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:07:15.775158683Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:07:16.406880256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:07:16.410528031Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:07:16.448830506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:07:16.452635382Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:07:16.769530932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:07:17.421171619Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T14:07:19.595447587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:07:19.59902031Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:07:21.334310062Z caller=checkpoint.go:378 msg="writing series" size="7.6 kB"
level=warn ts=2026-02-16T14:07:21.768767119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:07:21.958924595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:07:22.862220551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:07:22.866519293Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:07:23.18852379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:07:23.192080934Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:07:25.247117958Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T14:07:25.647858401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:07:25.651484292Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:07:26.768923502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:07:27.274740782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:07:29.159937514Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=info ts=2026-02-16T14:07:29.55898413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:07:29.562599531Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:07:31.768590234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:07:31.768623598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:07:31.768632532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:07:31.768628036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:07:33.073223254Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-02-16T14:07:34.355998101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:07:34.360726406Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:07:36.769004316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:07:36.986826432Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-02-16T14:07:37.022031708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:07:38.432373417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:07:38.43586914Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:07:40.347208153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:07:40.351357374Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:07:40.899904293Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=warn ts=2026-02-16T14:07:41.769083316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:07:43.571867482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:07:43.575936649Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:07:44.812671654Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-02-16T14:07:46.769347849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:07:47.860748021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:07:47.864699518Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:07:48.726007445Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=info ts=2026-02-16T14:07:49.667222788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:07:49.675056543Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:07:49.717140056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:07:49.721609977Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:07:50.389226525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:07:50.392745353Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:07:51.769363159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:07:52.646338611Z caller=checkpoint.go:378 msg="writing series" size="9.6 MB"
level=debug ts=2026-02-16T14:07:52.646365825Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=15
level=info ts=2026-02-16T14:07:54.905863405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:07:54.911759729Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T14:07:55.836253912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:57676"
level=info ts=2026-02-16T14:07:56.045056462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:07:56.049108788Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:07:56.551707748Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=warn ts=2026-02-16T14:07:56.768901833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:07:57.106903621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:07:57.111233617Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T14:07:57.2765477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:08:00.464607714Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=debug ts=2026-02-16T14:08:01.203146491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:08:01.748078993Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:08:01.748095774Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=warn ts=2026-02-16T14:08:01.769237185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:01.769248504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:08:01.769274832Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:08:01.769285741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:08:04.378607051Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:08:04.957973328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:08:04.96371026Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:08:05.245650516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:08:05.250229629Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:08:05.261286919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:08:05.265296441Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T14:08:06.768655755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:08:07.617342356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:08:08.290395953Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=info ts=2026-02-16T14:08:08.354818428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:08:08.359037366Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:08:09.928869158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:08:09.933569817Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:08:10.412789451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" idle=1 full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:08:10.417806377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:08:11.769036283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:08:12.20393204Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T14:08:13.836008746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:08:13.840150731Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:08:16.116869129Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T14:08:16.769309456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:18.789038089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.3 kB" total_uncomp="54 kB" avg_uncomp="27 kB" idle=1 full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:08:18.793096873Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:08:19.318238228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:08:19.322579486Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:08:19.390350308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:08:19.394844862Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:08:20.030092899Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=warn ts=2026-02-16T14:08:21.76883971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:22.842145234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:08:22.845912487Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:08:23.94422627Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-02-16T14:08:23.944256712Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=8
ts=2026-02-16T14:08:25.837678191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:47664"
level=warn ts=2026-02-16T14:08:26.7691508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:27.239204735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:08:27.247750059Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T14:08:27.278017496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:08:27.394962772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:08:27.399600348Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:08:27.856385706Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T14:08:28.111039267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:08:28.116148258Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:08:28.2325307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:08:28.236107985Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:08:31.768595226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:08:31.768607436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:08:31.768613258Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 kB" series=1
level=debug ts=2026-02-16T14:08:31.768624552Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:08:31.768633458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:08:31.847503389Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T14:08:31.850389204Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.082047651s
level=info ts=2026-02-16T14:08:33.648365432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:08:33.65200966Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:08:35.795307575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:08:35.798908957Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:08:36.7695165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:08:39.544337586Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:08:41.768505739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:42.183260238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:08:42.187453235Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:08:46.103622647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:08:46.11079545Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:08:46.76922303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:47.735528563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:08:47.740087156Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:08:47.74011474Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:08:47.740141553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:08:47.743651483Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:08:49.278835121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:08:49.282306174Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:08:51.361857317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:08:51.366186967Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:08:51.511366403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:08:51.76936104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:08:53.693776414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:08:53.697980186Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T14:08:56.768857717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:08:57.279835249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:08:57.813998854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:08:57.819312123Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:08:58.585526222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:08:58.589874043Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:08:58.999163946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:08:59.003248954Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:08:59.691258625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:08:59.695077734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:09:00.868351568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:09:00.87260708Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:09:01.747787448Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:09:01.748828224Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:09:01.767945118Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:09:01.768085196Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-02-16T14:09:01.768321934Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=info ts=2026-02-16T14:09:01.769042421Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:09:01.769054004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:09:01.76906315Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:09:01.769079421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:09:02.189294248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:09:02.193004344Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:09:04.365501474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:09:05.62649671Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-02-16T14:09:06.768680283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:09:09.483638249Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-02-16T14:09:11.76926884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:09:11.947784622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:09:11.952571048Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:09:12.197618002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:09:12.201485267Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:09:12.507474532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:09:12.511738106Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:09:12.846940964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:09:12.850743461Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:09:12.951166826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.3 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:09:12.955072987Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:09:13.340348957Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
level=info ts=2026-02-16T14:09:15.935747132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:09:15.939475349Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:09:16.481838805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:09:16.485562787Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:09:16.769348114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:09:17.197286101Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:09:17.741980659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:09:17.746273118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:09:17.746296825Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:09:17.746321574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:09:17.749733186Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:09:18.569310513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.4 kB" avg_uncomp="7.4 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:09:18.573371007Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:09:21.054666007Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
ts=2026-02-16T14:09:21.113014486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52512"
level=warn ts=2026-02-16T14:09:21.768538454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:09:24.911389388Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=debug ts=2026-02-16T14:09:24.911432525Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-02-16T14:09:26.768698034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:09:27.281374792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:09:28.768632222Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:09:29.522114828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:09:29.535767435Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:09:31.03846288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:09:31.044933905Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:09:31.131046269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:09:31.135009587Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:09:31.768771427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:09:31.768806807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:09:31.768818301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:09:31.768843806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:09:32.626151972Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
level=info ts=2026-02-16T14:09:33.736784546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:09:33.740649878Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:09:34.541740266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:09:34.546701795Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:09:36.270395908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:09:36.277762465Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:09:36.483470496Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-02-16T14:09:36.524721819Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:09:36.769134945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:09:40.110043071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:09:40.114143183Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:09:40.11416748Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:09:40.114189692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:09:40.118169407Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:09:40.340176911Z caller=checkpoint.go:378 msg="writing series" size="606 kB"
level=info ts=2026-02-16T14:09:40.436123545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:09:40.440527338Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:09:40.809761298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:09:40.813985251Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T14:09:41.768765507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:09:44.197344146Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T14:09:45.234370302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:09:45.238098958Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:09:45.816012648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:09:45.817943282Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T14:09:46.769423651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:09:48.042154713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:09:48.046654098Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:09:48.054767994Z caller=checkpoint.go:378 msg="writing series" size="510 B"
ts=2026-02-16T14:09:51.115558623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57662"
level=warn ts=2026-02-16T14:09:51.769084201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:09:51.911392656Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-02-16T14:09:53.647498296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.3 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:09:53.651587208Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
ts=2026-02-16T14:09:53.817740087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57674"
level=info ts=2026-02-16T14:09:54.33955504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:09:54.343914602Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:09:55.25641462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:09:55.260625278Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:09:55.511592713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:09:55.515577928Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:09:55.663152748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:09:55.667123791Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:09:55.768718846Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-02-16T14:09:56.686549171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:09:56.690428555Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:09:56.769240981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:09:56.983973137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:09:56.987797777Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:09:57.215781763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:09:57.220166517Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T14:09:57.282925296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:09:59.626640866Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=debug ts=2026-02-16T14:09:59.626671415Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-02-16T14:09:59.975176353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:09:59.979235526Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:10:01.442058835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:10:01.44774512Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:10:01.748053561Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:10:01.748392594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:10:01.768735192Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:10:01.768762141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:10:01.768771141Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:10:01.768843392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:10:03.482825288Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-02-16T14:10:04.588397219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:10:06.304045842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:10:06.308191971Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T14:10:06.769425242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:10:07.340026541Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T14:10:10.465047473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:10:10.46900153Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:10:10.937531275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:10:10.941986994Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:10:11.197770724Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-02-16T14:10:11.769368099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:12.264846909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:10:12.268693648Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:10:15.054357189Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-02-16T14:10:15.60887924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:10:15.612462477Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:10:16.769129669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:18.610196588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:10:18.610540165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:10:18.613848717Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:10:18.615577721Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:10:18.91166934Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-02-16T14:10:21.769113986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:22.263267553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:10:22.271624534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:10:22.773459599Z caller=checkpoint.go:378 msg="writing series" size="4.6 MB"
level=debug ts=2026-02-16T14:10:22.773486954Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.8 MB" series=6
level=debug ts=2026-02-16T14:10:26.625860393Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-02-16T14:10:26.768903967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:10:27.284440812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:10:27.440758755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:10:27.444573242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:10:30.483354446Z caller=checkpoint.go:378 msg="writing series" size="716 kB"
level=debug ts=2026-02-16T14:10:30.761220919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:10:30.761312246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="35 kB" avg_uncomp="35 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:10:30.765986911Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:10:31.768860102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:10:31.768896326Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:10:31.768907586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:10:31.768956342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:33.035151387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:10:33.038846381Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:10:33.109751031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:10:34.340542121Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:10:35.654626387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.3 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:10:35.658068039Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:10:36.671082718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:10:36.676345478Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:10:36.768751417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:37.562998465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:10:37.567935447Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:10:37.567965142Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:10:37.568000365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:10:37.571474029Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:10:37.762332015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:10:37.765711282Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:10:38.19730488Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T14:10:38.374842718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:10:38.378385444Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:10:40.299443146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:10:40.302882585Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:10:41.769375952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:42.044825495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:10:42.048464122Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:10:42.054573743Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-02-16T14:10:42.96501911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:10:42.968771266Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:10:43.369673774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:10:43.373354153Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:10:43.386540542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:10:43.390222542Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:10:44.143507414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:10:44.147087633Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:10:45.341921344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:10:45.345683879Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:10:45.5159288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:10:45.51953878Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:10:45.911941886Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-02-16T14:10:45.911967745Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-02-16T14:10:46.768801593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:10:49.768719776Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
level=warn ts=2026-02-16T14:10:51.768710341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:10:52.09862203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:10:52.102458418Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:10:53.090794699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:10:53.094190771Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:10:53.62278693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:10:53.625833753Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
level=error ts=2026-02-16T14:10:53.627652505Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:10:54.793671022Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T14:10:55.847445335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:37950"
level=warn ts=2026-02-16T14:10:56.769074931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:10:57.28544862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:10:57.483643494Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T14:10:57.97029689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:10:57.974647512Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:10:59.741848377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:10:59.74514598Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:11:01.340733751Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=debug ts=2026-02-16T14:11:01.747671931Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:11:01.748745406Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:11:01.768574135Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:11:01.768609911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:11:01.768618713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:11:01.768629655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:11:02.617844574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:11:02.622232551Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:11:05.197611039Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=warn ts=2026-02-16T14:11:06.76947983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:11:09.055026772Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=debug ts=2026-02-16T14:11:09.055054456Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-02-16T14:11:09.803249919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:11:09.80714671Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:11:10.164240871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.4 kB" avg_uncomp="5.4 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:11:10.168247228Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:11:11.768851823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:11:12.912143732Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-02-16T14:11:16.266190647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:11:16.270986789Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:11:16.372337915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:11:16.380706251Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:11:16.38073349Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:11:16.380761048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:11:16.388527817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:11:16.768642472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:11:16.768708883Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T14:11:19.57590669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:11:19.579751855Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:11:20.625951512Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:11:21.550741665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:11:21.554574855Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:11:21.768756139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:11:22.707768449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:11:22.711807668Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:11:23.000853351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.3 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:11:23.00432361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:11:24.483363003Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-02-16T14:11:24.999998953Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:11:25.514691101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:11:25.518085816Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T14:11:25.848682411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:39844"
level=info ts=2026-02-16T14:11:26.570343013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:11:26.573998983Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:11:26.768535568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:11:27.075529146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:11:27.078963439Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T14:11:27.28739169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:11:28.340033487Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-02-16T14:11:28.535659477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:11:28.539512628Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:11:29.485450617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:11:29.489010015Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:11:30.440525418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:11:30.447730408Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:11:31.089602418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:11:31.768434776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:11:31.768473229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:11:31.76848454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:11:31.768550192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:11:32.19753047Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:11:33.800991802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:11:33.804797132Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:11:35.609537119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:11:35.614923777Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:11:35.929923855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:11:35.933315298Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:11:36.054981756Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=warn ts=2026-02-16T14:11:36.769237016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:11:37.247792383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:11:37.251739901Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:11:37.37207954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:11:37.375964167Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:11:39.587897932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:11:39.591984702Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:11:39.911282745Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T14:11:41.769105523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:11:42.970225524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:11:42.974501382Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:11:43.768979179Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:11:45.775971033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:11:45.780054912Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:11:46.40901796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:11:46.412538782Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:11:46.769502034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:11:47.625529463Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-02-16T14:11:51.483083952Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T14:11:51.768817663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:11:55.340653418Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
level=debug ts=2026-02-16T14:11:56.767707176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:11:56.768766247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:11:57.289362154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:11:57.346327839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:11:57.356222105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:11:57.387215766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:11:57.39116664Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:11:59.197851716Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T14:12:01.748205145Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:12:01.74823342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:12:01.769113354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:12:01.769145104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:12:01.769155177Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:12:01.769171098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:12:03.054846965Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-02-16T14:12:04.744205519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:12:04.749783062Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:12:06.768773711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:12:06.912178731Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:12:08.034454128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:12:08.038965167Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:12:08.69671868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="391 kB" avg_uncomp="196 kB" idle=1 full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:12:08.700632983Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:12:10.348212999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:12:10.352047742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:12:10.768604076Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-02-16T14:12:10.76863639Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=info ts=2026-02-16T14:12:11.764102597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:12:11.767492787Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:12:11.768449554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:12:12.179876246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.3 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:12:12.18332753Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66bb4af0%3A19c66bb4af2%3A71b44155?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:12:12.183353118Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:12:12.183387524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" idle=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:12:12.194142877Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:12:12.494928039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:12:12.498786066Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:12:14.62587731Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:12:14.937691344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:12:14.941545503Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:12:14.941566044Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:12:15.242239023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:12:15.245768253Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:12:16.769057426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:12:17.881308729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:12:17.885225335Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:12:17.987611565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:12:17.991446416Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:12:17.99146869Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:12:17.99150294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:12:17.994537512Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:12:18.482757077Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=debug ts=2026-02-16T14:12:18.515944479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:12:19.109465748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:12:19.113290116Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:12:20.584674024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:12:20.588779763Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:12:21.768904777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:12:22.178393741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:12:22.18443576Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:12:22.340037922Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=info ts=2026-02-16T14:12:24.33017838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:12:24.335067243Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:12:24.726146917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:12:24.729617157Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:12:26.107099811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:12:26.110829288Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:12:26.110855604Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:12:26.110888144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:12:26.114082312Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:12:26.197488949Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T14:12:26.76939139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:12:27.052058323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:12:27.055344619Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T14:12:27.298417744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:12:27.314046496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:12:27.318097953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:12:29.35439069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:12:29.358290277Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:12:29.641171157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:12:29.645227582Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:12:30.054058425Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-02-16T14:12:31.768547423Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:12:31.768582583Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:12:31.76859114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:12:31.768598348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:12:31.886953526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:12:32.518134904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:12:32.521876867Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:12:32.646979651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:12:32.654367345Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:12:33.911991304Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=warn ts=2026-02-16T14:12:36.768701895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:12:37.769169064Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-02-16T14:12:37.948030354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:12:37.952043213Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:12:39.740082729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:12:39.745293553Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:12:41.62572799Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=warn ts=2026-02-16T14:12:41.768612541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:12:43.098304337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:12:43.102239774Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:12:43.102259861Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:12:43.102280474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:12:43.105466796Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:12:43.607997847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:12:43.611953269Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:12:45.482976919Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=warn ts=2026-02-16T14:12:46.768797162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:12:47.510151799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:12:47.51906737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:12:49.340127318Z caller=checkpoint.go:378 msg="writing series" size="494 kB"
level=debug ts=2026-02-16T14:12:49.34015446Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T14:12:50.917906982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:12:50.921674091Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T14:12:51.128545506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56492"
level=warn ts=2026-02-16T14:12:51.769263281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:12:53.119156469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:12:53.15268921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:12:53.156039612Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:12:53.206544199Z caller=checkpoint.go:378 msg="writing series" size="15 MB"
level=debug ts=2026-02-16T14:12:53.206579958Z caller=checkpoint.go:390 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-02-16T14:12:54.353372294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:12:54.35657042Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:12:54.386077167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:12:54.389728602Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:12:56.768905927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:12:57.05456225Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
ts=2026-02-16T14:12:57.299962117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:13:00.911692591Z caller=checkpoint.go:378 msg="writing series" size="265 kB"
level=info ts=2026-02-16T14:13:01.748109088Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:13:01.748131659Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:13:01.768524524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:13:01.768555677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:13:01.768565251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:13:01.768568511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:13:03.135479245Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:13:04.768459912Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:13:05.139516539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:13:05.144520845Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:13:05.837329594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:13:05.841066642Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:13:05.841088574Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:05.841110001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:13:05.844140737Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:13:06.768945562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:07.739882997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:13:07.743576346Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:13:07.743609279Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:07.743636732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="30 kB" avg_comp="15 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:13:07.74670735Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:13:08.193271119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:13:08.197317287Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:13:08.626540167Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=info ts=2026-02-16T14:13:09.929142397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:13:09.932956339Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:13:09.932980232Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:09.93300534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:13:09.935747058Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:13:10.250704966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:13:10.254107438Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:13:10.254127384Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:10.254154941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:13:10.257494574Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:13:10.266572854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:13:10.269637753Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:13:10.269654816Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:10.554585484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:13:10.557951658Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:13:10.855995054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:13:10.859439701Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66bb381e%3A19c66bb381f%3Ae2bf5e6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:13:10.85945824Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:10.859479322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="299 kB" avg_uncomp="299 kB" idle=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:13:10.862325004Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:13:11.769151644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:12.281394672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:13:12.284755868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:13:12.284776875Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:12.284799917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:13:12.287481911Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:13:12.482782287Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T14:13:14.017738799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:13:14.020900507Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:13:14.02092454Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:14.02094816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:13:14.023689429Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:13:15.591048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:13:15.594580926Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:13:15.594609348Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:15.594633109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:13:15.59738304Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:13:16.137224734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:13:16.140422043Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:13:16.339846995Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T14:13:16.768845332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:16.80500615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:13:16.808805069Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:13:18.798312149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:13:18.802033401Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:13:18.802055113Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:18.802079244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:13:18.804986936Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:13:20.197712051Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T14:13:21.638826719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:13:21.642520629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:13:21.642545066Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:21.642569932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:13:21.645514342Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T14:13:21.768668121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:22.498884451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:13:22.502422044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:13:23.829486248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49656"
level=debug ts=2026-02-16T14:13:24.055602521Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-02-16T14:13:24.055627241Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=8
level=info ts=2026-02-16T14:13:24.249332276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:13:24.252839696Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:13:24.652460653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:13:24.655846264Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:13:25.260979613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:13:25.264975596Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:13:25.368348213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:13:25.372197516Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:13:26.769158626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:13:27.301289301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:13:27.568026482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:13:27.571805305Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:13:27.626852135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:13:27.630553727Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:13:27.911766241Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:13:29.064703755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:13:29.06840011Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66bb4af0%3A19c66bb4af2%3A9721fa47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T14:13:29.068438851Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:13:29.068461157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="97 kB" avg_uncomp="97 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:13:29.071568484Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:13:29.137781636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:13:29.141391964Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:13:29.647117994Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:13:31.306850637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:13:31.310057297Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:13:31.768495792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:13:31.768527412Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:13:31.768536005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:13:31.768545145Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 kB" series=1
level=warn ts=2026-02-16T14:13:31.768549351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:31.771839872Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-02-16T14:13:31.775118209Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006900125s
level=info ts=2026-02-16T14:13:32.012389259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:13:32.015723175Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:13:32.338691773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:13:32.341965895Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:13:33.832430882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:13:33.835684731Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:13:34.058318262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:13:34.061634026Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:13:36.769455867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:37.33388914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:13:37.3375815Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:13:39.423872166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:13:39.425837184Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:13:39.65586244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:13:39.659596775Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T14:13:41.769336803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:41.792461093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:13:41.797713838Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:13:41.844927674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:13:41.848709727Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:13:43.430207464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:13:43.437778422Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:13:46.425199173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:13:46.428959033Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:13:46.769435363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:47.637180197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:13:47.641894196Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:13:49.129800012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:13:49.133587315Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T14:13:51.132534297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34054"
level=info ts=2026-02-16T14:13:51.670906678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:13:51.675515988Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T14:13:51.768652334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:13:52.100722112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:13:52.104597777Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:13:55.227797551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:13:55.231633889Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:13:55.782430719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:13:55.786388082Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:13:56.768949227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:13:57.303019472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:14:00.935534365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:14:00.940255275Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:14:01.748055479Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:14:01.748436941Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:14:01.767873328Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:14:01.768064087Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-02-16T14:14:01.768375521Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=info ts=2026-02-16T14:14:01.768428229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:14:01.768452929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:14:01.768460961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:14:01.768456192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:01.955859984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:14:01.959196122Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:14:01.997289407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:14:02.000773288Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:14:02.530275628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:14:02.533732921Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:14:02.691942903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:14:02.695891146Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:14:03.162804778Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:14:04.321357045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:14:04.324796826Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:14:04.741473271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:14:04.754621585Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:14:05.004070387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:14:05.00771133Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:14:05.096921386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:14:05.101728455Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:14:05.572120994Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=warn ts=2026-02-16T14:14:06.771639154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:14:09.374975697Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T14:14:09.565162589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:14:09.568883489Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:14:11.768973532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:12.376215647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:14:12.379689443Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:14:13.177849708Z caller=checkpoint.go:378 msg="writing series" size="657 kB"
level=warn ts=2026-02-16T14:14:16.768600032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:14:16.980211418Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:14:17.823109004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:14:17.82817065Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:14:20.783215088Z caller=checkpoint.go:378 msg="writing series" size="379 kB"
level=debug ts=2026-02-16T14:14:20.783244552Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-02-16T14:14:21.134956688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47204"
level=warn ts=2026-02-16T14:14:21.768739232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:23.464269597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:14:23.468823732Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:14:24.310738871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:14:24.314856618Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:14:24.585351888Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-02-16T14:14:26.76933966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:14:27.304892144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:14:27.689788944Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:14:28.388760351Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:14:31.768788047Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:14:31.768788969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:14:31.768821706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:14:31.768831469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:14:32.191135298Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
level=debug ts=2026-02-16T14:14:35.245603375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:14:35.523292118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:14:35.528028773Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:14:35.915226733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=info ts=2026-02-16T14:14:35.916302333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:14:35.918684842Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:14:35.918701627Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:14:35.994175669Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T14:14:36.769165882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:38.433595705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:14:38.440242287Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:14:39.797214881Z caller=checkpoint.go:378 msg="writing series" size="438 kB"
level=info ts=2026-02-16T14:14:39.836518868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:14:39.841173369Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T14:14:41.769142324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:42.04306407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:14:42.051488578Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:14:43.600303271Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:14:43.600330038Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=6
level=info ts=2026-02-16T14:14:44.183863857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:14:44.187866852Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:14:45.014726168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:14:45.018839824Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:14:45.228536815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:14:45.232391653Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:14:45.75273638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:14:45.756703239Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:14:46.768658241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:14:47.402784243Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T14:14:49.820898637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:14:49.826152026Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:14:49.831146701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:14:49.834662693Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:14:51.13708692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46200"
level=debug ts=2026-02-16T14:14:51.205914267Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=warn ts=2026-02-16T14:14:51.768613226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:52.831457752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:14:52.835195639Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:14:53.374855079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:14:53.378516454Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:14:54.106540807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:14:54.110001425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:14:55.008251786Z caller=checkpoint.go:378 msg="writing series" size="8.5 kB"
level=info ts=2026-02-16T14:14:55.739454413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:14:55.74282779Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:14:56.768736707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:14:56.972572674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:14:56.97642612Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T14:14:57.306515475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:14:58.811457508Z caller=checkpoint.go:378 msg="writing series" size="528 kB"
level=info ts=2026-02-16T14:14:59.485315752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:14:59.489063914Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:14:59.904223581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:14:59.907910046Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:14:59.907933041Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:14:59.907957541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:14:59.910974073Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:15:01.748161547Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:15:01.748363579Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:15:01.748600588Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T14:15:01.748644885Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968056 n=1
level=debug ts=2026-02-16T14:15:01.748692286Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968056 n=1
level=info ts=2026-02-16T14:15:01.768783254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:15:01.768812454Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:15:01.768823146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:15:01.768824401Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:15:02.613692638Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:15:02.624808547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:15:02.628784461Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:15:02.87281471Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:15:02.872893915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:15:02.878637241Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:15:03.952996164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:15:03.956609781Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:15:06.416625132Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T14:15:06.768548063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:15:07.074334959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:15:07.079061257Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:15:10.220169442Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-02-16T14:15:11.769244284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:15:12.494537336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:15:12.49846791Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:15:14.0223142Z caller=checkpoint.go:378 msg="writing series" size="203 kB"
level=warn ts=2026-02-16T14:15:16.768484851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:15:17.677365222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:15:17.684774027Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:15:17.825348265Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=info ts=2026-02-16T14:15:18.02776427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:15:18.031431003Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:15:18.630359571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:15:18.634204897Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T14:15:21.139316045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52246"
level=debug ts=2026-02-16T14:15:21.631226853Z caller=checkpoint.go:378 msg="writing series" size="5.1 MB"
level=debug ts=2026-02-16T14:15:21.631252825Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.1 MB" series=10
level=warn ts=2026-02-16T14:15:21.768596351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:15:22.313474405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:15:22.318023858Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:15:24.989683951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:15:24.993666134Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:15:25.332450762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:15:25.33610378Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:15:25.43085571Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T14:15:25.862463536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:60154"
level=warn ts=2026-02-16T14:15:26.76853479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:15:27.308394623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:15:29.14509976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:15:29.233538724Z caller=checkpoint.go:378 msg="writing series" size="754 kB"
level=info ts=2026-02-16T14:15:29.502420305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:15:29.50609211Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:15:31.76889886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:15:31.768934607Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:15:31.768942687Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:15:31.768953242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:15:31.902284995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:15:31.905875736Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:15:33.036827643Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:15:35.514578762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:15:35.518218156Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:15:35.761308469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:15:35.76481679Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:15:36.769415613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:15:36.839743062Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T14:15:37.016397495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:15:37.019925618Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:15:37.197276852Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:15:38.171420358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:15:38.175015858Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:15:38.665724656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:15:38.672071427Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:15:40.230439527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:15:40.236180462Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:15:40.536029942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:15:40.543398089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:15:40.641672532Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-02-16T14:15:41.769175784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:15:42.032178224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:15:42.03593292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:15:43.116872755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:15:43.120312077Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:15:44.435168112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:15:44.438696313Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:15:44.444974169Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=debug ts=2026-02-16T14:15:44.444988741Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-02-16T14:15:45.956844964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:15:45.960526735Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:15:46.007617174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:15:46.011041284Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:15:46.238791164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:15:46.242559997Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:15:46.768596879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:15:48.248069729Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=info ts=2026-02-16T14:15:51.425171786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:15:51.428927171Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:15:51.769005073Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:15:52.05116593Z caller=checkpoint.go:378 msg="writing series" size="814 kB"
level=debug ts=2026-02-16T14:15:52.05119569Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-02-16T14:15:52.865129975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:15:52.870316452Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:15:52.870341442Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:15:52.870367094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:15:52.873768116Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:15:53.221828573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:15:53.225556002Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T14:15:53.838623624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49270"
level=info ts=2026-02-16T14:15:54.764270837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="461 B" avg_comp="461 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:15:54.767988502Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:15:55.85306643Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-02-16T14:15:56.769194164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:15:57.30979705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:15:59.656041167Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=info ts=2026-02-16T14:16:00.688236318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:16:00.691972301Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:16:00.691996253Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:16:00.692017726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:16:00.694805393Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:16:01.508884731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:16:01.512233223Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:16:01.748044593Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:16:01.74843494Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:16:01.768709463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:16:01.768740778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:16:01.768749182Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:16:01.768748443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:16:03.459154778Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=warn ts=2026-02-16T14:16:06.769458416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:16:06.783720817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:16:07.262124194Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=debug ts=2026-02-16T14:16:11.064499988Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=info ts=2026-02-16T14:16:11.767474735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:16:11.768572446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-02-16T14:16:11.771769283Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:16:12.075634894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:16:12.079819997Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:16:14.867222413Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:16:15.187193379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:16:15.191594116Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T14:16:16.768918482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:16:17.328530665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:16:17.334166493Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:16:18.670076451Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:16:19.369297235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:16:19.372743485Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:16:20.239193608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:16:20.242557319Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T14:16:21.14357443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:59218"
level=info ts=2026-02-16T14:16:21.604745127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:16:21.610199224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:16:21.76872432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:16:21.882001651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:16:21.885327508Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:16:22.473636315Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:16:23.049457333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:16:23.053017715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:16:23.248965747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:16:24.882081677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:16:24.88573334Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:16:25.76252284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:16:25.769138156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:16:26.276205646Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=warn ts=2026-02-16T14:16:26.768691915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:16:27.311616983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:16:28.554583283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:16:28.558475879Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:16:30.078392063Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:16:30.678219357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:16:30.682331739Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:16:30.778829723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:16:30.782525091Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:16:31.443776033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:16:31.447567054Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:16:31.768826122Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:16:31.768854138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:16:31.768861749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:16:31.768883215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:16:33.881304925Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-02-16T14:16:36.768896218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:16:36.829051434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" idle=1 full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:16:36.833265196Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:16:36.888423404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:16:36.891999304Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:16:37.080103926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:16:37.083919073Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:16:37.685060856Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=info ts=2026-02-16T14:16:38.688941694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:16:38.692524085Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:16:38.706605992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:16:38.710340345Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:16:39.321224641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:16:39.324650485Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:16:40.35302757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:16:41.487233968Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-02-16T14:16:41.769390798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:16:44.863980443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:16:44.869254008Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:16:45.264430801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:16:45.268012326Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:16:45.290133688Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:16:45.523766532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:16:45.527385649Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:16:46.768477801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:16:47.189654313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:16:47.19330571Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:16:49.092738282Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T14:16:51.768928271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:16:52.078837344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:16:52.083395059Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:16:52.895448025Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-02-16T14:16:52.895474663Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
ts=2026-02-16T14:16:53.843049992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39456"
level=debug ts=2026-02-16T14:16:56.698494764Z caller=checkpoint.go:378 msg="writing series" size="484 kB"
level=warn ts=2026-02-16T14:16:56.768689978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:16:57.313436942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:17:00.469009838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:17:00.472936443Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:17:00.472958786Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:17:00.472981976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:17:00.475881367Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:17:00.501053646Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=info ts=2026-02-16T14:17:01.748021928Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:17:01.748022478Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:17:01.769347591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:17:01.769388898Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:17:01.769398946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:17:01.769436993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:17:02.514356043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:17:02.520358485Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:17:04.304358907Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-02-16T14:17:04.748904491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:17:04.752267461Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:17:06.769507005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:17:06.775601714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:17:06.781811519Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:17:07.12163913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:17:07.488890204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:17:07.492718931Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:17:07.799759184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:17:07.803447079Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:17:08.107073544Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:17:11.339322844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:17:11.343100246Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:17:11.769232018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:17:11.909799442Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=debug ts=2026-02-16T14:17:11.909824078Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-02-16T14:17:12.010060212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:17:12.013696715Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:17:12.920729137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:17:12.924863079Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:17:13.625189708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:17:15.237396182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:17:15.241653571Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:17:15.712874779Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T14:17:16.45052127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:17:16.454158675Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:17:16.769015111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:17:17.4902202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="569 B" avg_comp="284 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:17:17.494034056Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:17:18.375450631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:17:18.378953283Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:17:19.515570313Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-02-16T14:17:21.136805799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:17:21.140353311Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:17:21.147303809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38544"
level=warn ts=2026-02-16T14:17:21.768885387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:17:23.317888384Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
ts=2026-02-16T14:17:23.844752915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:45092"
level=info ts=2026-02-16T14:17:24.316655136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="1.9 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:17:24.320610928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T14:17:26.769329044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:17:27.121056299Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
ts=2026-02-16T14:17:27.315517941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:17:28.239797706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:17:28.243865851Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:17:30.058631363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:17:30.062556444Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:17:30.923939295Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=info ts=2026-02-16T14:17:31.0635248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:17:31.066868388Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:17:31.768969819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:17:31.76896374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:17:31.768994989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:17:31.769006394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:17:32.065072867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:17:32.068528343Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:17:32.392422122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:17:32.395749986Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:17:34.494369594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:17:34.498005816Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:17:34.7264711Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
level=warn ts=2026-02-16T14:17:36.769149693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:17:37.433290244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:17:37.437998989Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:17:37.438027373Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:17:37.438054388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:17:37.441706648Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:17:38.529615238Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-02-16T14:17:38.878311678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:17:38.881988357Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:17:40.757904623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" idle=1 full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:17:40.761455265Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:17:41.769397286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:17:42.332536625Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:17:42.349677901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:17:42.353085002Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:17:42.993968617Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:17:44.029336962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:17:44.033032615Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:17:45.701637313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:17:45.709263304Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:17:46.135680046Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-02-16T14:17:46.768694308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:17:49.937957362Z caller=checkpoint.go:378 msg="writing series" size="675 kB"
level=debug ts=2026-02-16T14:17:49.937983658Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-02-16T14:17:50.560019328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:17:50.564062385Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:17:51.769436199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:17:53.21319904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:17:53.217767389Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:17:53.764262007Z caller=checkpoint.go:378 msg="writing series" size="19 MB"
level=debug ts=2026-02-16T14:17:53.764292055Z caller=checkpoint.go:390 msg="flushing series" totalSize="19 MB" series=1
level=info ts=2026-02-16T14:17:55.054809361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:17:55.058437862Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T14:17:56.769215797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:17:57.317712048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:17:57.543280107Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=info ts=2026-02-16T14:17:57.601347846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:17:57.608568897Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:17:57.60860073Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:17:57.608630004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:17:57.615128128Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:18:01.102291408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:18:01.103831378Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:18:01.34601822Z caller=checkpoint.go:378 msg="writing series" size="466 kB"
level=debug ts=2026-02-16T14:18:01.747755757Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:18:01.748851219Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:18:01.76919136Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:18:01.769225836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T14:18:01.769226857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:01.769238062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:18:03.25341668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:18:03.25681693Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:18:03.617799197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:18:03.621312281Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:18:04.627818904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:18:04.631365121Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:18:05.148994359Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:18:06.428668955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:18:06.432361339Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:18:06.498556688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:18:06.501899702Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:18:06.769459384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:07.323447919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:18:07.326858671Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:18:08.756466292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:18:08.763598073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:18:08.95231385Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=info ts=2026-02-16T14:18:09.556499397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:18:09.560349647Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:18:11.768913522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:18:12.754424341Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-02-16T14:18:14.627639195Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:18:14.627735228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="210 kB" avg_comp="210 kB" total_uncomp="833 kB" avg_uncomp="833 kB" idle=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:18:14.632243931Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:18:15.426299636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:18:15.429669744Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:18:16.259182115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:18:16.263667821Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:18:16.557647931Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T14:18:16.588789149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:18:16.592385046Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:18:16.768852087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:18.624067831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:18:18.628181304Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:18:18.913156424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:18:18.916893349Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:18:18.916922181Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:18:18.916953226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:18:18.920084966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:18:19.188058671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:18:19.191717661Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:18:20.360330725Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=warn ts=2026-02-16T14:18:21.769152814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:21.776206141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:18:21.77969062Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T14:18:23.848489276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39046"
level=debug ts=2026-02-16T14:18:24.16452564Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T14:18:24.164551961Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=8
level=info ts=2026-02-16T14:18:24.603439305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:18:24.609595794Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
ts=2026-02-16T14:18:25.872298352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:56928"
level=warn ts=2026-02-16T14:18:26.769298017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:18:27.319670911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:18:27.966045861Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:18:30.940392601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:18:30.944330857Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:18:31.768882298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:31.768893995Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:18:31.768923122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:18:31.768931459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:18:31.768920515Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.2 kB" series=1
level=info ts=2026-02-16T14:18:31.773850664Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-02-16T14:18:31.776092517Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007821317s
level=info ts=2026-02-16T14:18:31.940715173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:18:31.944782128Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:18:32.852115859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:18:32.856008608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:18:33.140690772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:18:33.144427714Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:18:33.756341026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:18:33.760496382Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:18:34.17327571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:18:34.176667823Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:18:36.768869112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:36.874229947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:18:36.87811848Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:18:36.878140194Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:18:36.878163962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:18:36.881431867Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:18:41.109866937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:18:41.113572408Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:18:41.769221954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:43.616198875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:18:43.622558349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:18:44.957861618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:18:44.962148201Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:18:45.647156869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:18:45.653960977Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:18:46.768999099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:18:48.8828994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:18:48.889126148Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:18:49.519838584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:18:49.519835921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="71 kB" avg_comp="71 kB" total_uncomp="384 kB" avg_uncomp="384 kB" idle=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:18:49.530564425Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:18:50.234932819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:18:50.238805541Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:18:50.74212427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:18:50.746020146Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T14:18:51.154548199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34350"
level=warn ts=2026-02-16T14:18:51.76855458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:18:53.849773575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:44394"
level=info ts=2026-02-16T14:18:55.766470065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:18:55.769991502Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:18:56.769478564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:18:57.321239708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:18:57.865172285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:18:58.518938998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:18:58.523857516Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:18:58.523881631Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:18:58.52390508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:18:58.526872631Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:18:58.729544355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:18:58.732956784Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:18:59.733094521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:18:59.736855622Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:19:00.477865406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:19:00.481633323Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:19:01.663020825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:19:01.66650019Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:19:01.747482417Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:19:01.74854561Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:19:01.76787984Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:19:01.768028772Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-02-16T14:19:01.768280683Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=info ts=2026-02-16T14:19:01.768986559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:19:01.768999177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:19:01.769006959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:19:01.768999072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:02.969853949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:19:02.973491396Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:19:04.239281303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:19:04.244399504Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:19:05.007714765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:19:05.011971345Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:19:05.572111638Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-02-16T14:19:06.769145291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:07.284621814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:19:07.288318869Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:19:09.37416167Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T14:19:10.425010544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:19:10.428528297Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:19:10.428552794Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:10.428577069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:19:10.431741585Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:19:11.769499137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:19:13.178925046Z caller=checkpoint.go:378 msg="writing series" size="960 kB"
level=debug ts=2026-02-16T14:19:13.178958944Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T14:19:13.285374368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:19:13.288999454Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:19:14.012528483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:19:14.015834383Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:19:14.482327848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:19:14.485955484Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:19:14.613110107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:19:14.6172938Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:19:15.320177066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:19:15.324221907Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:19:15.324252936Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:15.324285325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:19:15.32773313Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:19:15.579687158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:19:15.583265703Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:19:16.769198288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:16.909526392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:19:16.913598349Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:19:16.980243499Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:19:17.504049176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:19:17.508692282Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:19:18.948965849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:19:18.953208097Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:19:19.48746404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:19:19.49103252Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:19:19.890628243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:19:20.782922803Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=warn ts=2026-02-16T14:19:21.768904616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:22.630066828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:19:22.63383277Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:19:24.585770124Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-02-16T14:19:26.768879662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:19:27.322725715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:19:28.388860843Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:19:28.759130295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:19:28.764119004Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:19:28.955743195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:19:28.95989388Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:19:31.768506261Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:19:31.768536461Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:19:31.768545058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:19:31.768540648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:19:32.19197733Z caller=checkpoint.go:378 msg="writing series" size="350 kB"
level=info ts=2026-02-16T14:19:33.221298768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:19:33.225429345Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:19:35.994600496Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T14:19:36.150182274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:19:36.159797889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:19:36.76887771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:38.788889655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:19:38.797558578Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:19:39.454015261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:19:39.458239144Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:19:39.45826118Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:39.458284049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:19:39.461455283Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:19:39.797648938Z caller=checkpoint.go:378 msg="writing series" size="464 kB"
level=debug ts=2026-02-16T14:19:39.797671927Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-02-16T14:19:39.939854855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:19:39.944370472Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:19:41.768675615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:42.31927031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:19:42.323098756Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:19:42.323120069Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:42.624025336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:19:42.627596572Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:19:43.600512093Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-02-16T14:19:43.600537431Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-02-16T14:19:43.882162103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:19:43.885721929Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:19:43.885743673Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:43.885764542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:19:43.888577889Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:19:43.981986685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:19:43.985206428Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:19:44.720855124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:19:44.724144112Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:19:45.025897893Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:19:45.327685059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:19:45.330866975Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:19:46.76887479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:47.215119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:19:47.218662399Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:19:47.218696995Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:47.218730885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:19:47.221936997Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:19:47.402171689Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T14:19:48.472395853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:19:48.476302636Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:19:48.476328265Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:48.476352279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:19:48.479917355Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:19:49.235197049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:19:49.240979506Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:19:49.241010084Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:49.241035866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:19:49.244492303Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:19:50.317861039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:19:50.3216483Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:19:50.321674961Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:50.321700507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:19:50.325091349Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:19:51.205896209Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-02-16T14:19:51.444693806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:19:51.448572728Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:19:51.706818396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:19:51.768937858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:54.367719115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:19:54.371198799Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:19:55.007840582Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=info ts=2026-02-16T14:19:56.728318785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:19:56.732152749Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:19:56.768880627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:19:56.931268063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:19:56.934683277Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:19:56.934704953Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:56.934726649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:19:56.937479989Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T14:19:57.324481316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:19:57.591345559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:19:57.595121137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:19:57.595147401Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:19:57.595171848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:19:57.597905717Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:19:57.602963711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:19:57.605803837Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:19:57.738110972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:19:57.742113467Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:19:58.811235178Z caller=checkpoint.go:378 msg="writing series" size="492 kB"
level=info ts=2026-02-16T14:20:00.679718688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:20:00.684684976Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:20:00.684716743Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:20:00.684750049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:20:00.688104795Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:20:00.980013356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:20:00.985147852Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:20:00.985177259Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:20:00.98521067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:20:00.988553542Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:20:01.748081523Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:20:01.748506569Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:20:01.768636966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:20:01.768648785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:20:01.768671915Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:20:01.768691929Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:20:02.613876285Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:20:03.057546203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:20:03.061156169Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:20:03.839343614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:20:03.843487229Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:20:05.274624317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:20:05.278539511Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:20:05.449088046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:20:05.45317302Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:20:05.64271001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:20:05.646694916Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:20:06.00383507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:20:06.007740814Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:20:06.007770269Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:20:06.007804744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:20:06.011102859Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:20:06.41617938Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T14:20:06.769531795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:20:08.127761861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:20:08.131595887Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:20:09.278746338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:20:09.282911901Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:20:09.49267444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:20:09.496665991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:20:10.219759248Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-02-16T14:20:11.769145241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:20:12.046263169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:20:12.050589617Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:20:12.916252829Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:20:14.022758726Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T14:20:14.959180522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:20:14.963198142Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:20:15.551871034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:20:15.556709298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:20:16.752744328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:20:16.756500699Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:20:16.768454497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:20:17.273004001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:20:17.276662468Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:20:17.465423898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:20:17.468707869Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:20:17.639348292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:20:17.642997466Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:20:17.824676119Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T14:20:19.459018445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:20:19.463190617Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T14:20:21.161200701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35388"
level=info ts=2026-02-16T14:20:21.478671697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:20:21.482219026Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:20:21.63091749Z caller=checkpoint.go:378 msg="writing series" size="5.9 MB"
level=debug ts=2026-02-16T14:20:21.630946881Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.9 MB" series=10
level=warn ts=2026-02-16T14:20:21.769238015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:20:22.735800408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:20:22.742828795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:20:23.704852232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:20:23.710998615Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T14:20:23.85537147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:38006"
level=info ts=2026-02-16T14:20:25.128942734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:20:25.132415647Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:20:25.430265177Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:20:25.47355757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:20:25.47698675Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:20:26.110774158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:20:26.114457042Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:20:26.768585965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:20:26.946290509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:20:26.950001953Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T14:20:27.326378268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:20:28.178021176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:20:28.181749942Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:20:28.729160553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:20:28.732746946Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:20:29.043214877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:20:29.234307076Z caller=checkpoint.go:378 msg="writing series" size="734 kB"
level=info ts=2026-02-16T14:20:31.769189485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:20:31.769217129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:20:31.769224971Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:20:31.769221762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:20:33.036218881Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:20:36.198020175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:20:36.203083614Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:20:36.76845981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:20:36.838805305Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-02-16T14:20:36.912938648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:20:36.916239846Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:20:39.326844393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:20:39.330880304Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:20:39.660366791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:20:39.664445797Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:20:40.040823568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:20:40.044691606Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:20:40.642468217Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-02-16T14:20:41.768562444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:20:43.299071714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:20:43.302730244Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:20:44.445488376Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=debug ts=2026-02-16T14:20:44.445514072Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-02-16T14:20:46.237557372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:20:46.241224776Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:20:46.40661911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:20:46.410140674Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:20:46.528431956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:20:46.532075428Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:20:46.768705526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:20:48.05248842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:20:48.198835976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:20:48.202471981Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:20:48.247897773Z caller=checkpoint.go:378 msg="writing series" size="372 kB"
level=warn ts=2026-02-16T14:20:51.768584111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:20:52.05081983Z caller=checkpoint.go:378 msg="writing series" size="912 kB"
level=debug ts=2026-02-16T14:20:52.050845219Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-02-16T14:20:53.407782464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:20:53.4115046Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:20:55.853241457Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-02-16T14:20:56.769456038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:20:57.327441362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:20:57.4747473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:20:57.479533457Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:20:59.655905255Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=info ts=2026-02-16T14:21:01.748043739Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:21:01.748319454Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:21:01.768695547Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:21:01.768703079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:21:01.76872151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:21:01.76873076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:21:02.263830585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:21:02.26784279Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:21:03.458609884Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=info ts=2026-02-16T14:21:03.721199106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:21:03.728776946Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:21:05.25359792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:21:05.257435893Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:21:05.964682859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:21:05.970798528Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:21:05.970824778Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:21:05.970848969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:21:05.976101274Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:21:06.769001102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:21:07.261643122Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T14:21:10.612459939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:21:10.617472736Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:21:11.064935569Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-02-16T14:21:11.768648903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:21:12.064969519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:21:12.068561941Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:21:12.915541811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:21:12.919040509Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:21:13.863096225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:21:13.866588265Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:21:14.029073731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:21:14.033446231Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:21:14.180544102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:21:14.184632981Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:21:14.867094351Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T14:21:16.768944166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:21:16.886287469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="44 kB" avg_comp="44 kB" total_uncomp="299 kB" avg_uncomp="299 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:21:16.891340777Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:21:16.907381976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:21:16.913145063Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:21:17.849448188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:21:17.852821825Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:21:18.284171316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:21:18.670189293Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:21:19.1134496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:21:19.116840542Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:21:20.518822427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:21:20.522507786Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:21:21.769122937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:21:22.472561629Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:21:23.618836293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:21:23.622913839Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:21:24.210779756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:21:24.214647121Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:21:24.834548188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:21:24.837996458Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T14:21:25.880568721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:54948"
level=debug ts=2026-02-16T14:21:26.275993275Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=warn ts=2026-02-16T14:21:26.768771723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:21:27.329284719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:21:27.54080924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:21:27.545060078Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:21:29.127995126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:21:29.917061185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:21:29.920775097Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:21:30.078389286Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:21:30.199789664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:21:30.203238688Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:21:30.977527512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:21:30.981034759Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:21:31.768993234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:21:31.769024569Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:21:31.769034431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:21:31.769034092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:21:32.584045847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="587 B" avg_comp="293 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:21:32.587591948Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:21:33.881352798Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:21:34.898028774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:21:34.901458316Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:21:34.901481132Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:21:34.901505134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:21:34.904269884Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:21:36.76856214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:21:37.684388945Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=info ts=2026-02-16T14:21:39.003625208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:21:39.007474164Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:21:41.486989708Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-02-16T14:21:41.768542577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:21:42.695195274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:21:42.699210272Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:21:43.771766513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:21:43.778450092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:21:45.28954958Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:21:45.71673202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:21:45.720776403Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:21:46.768609752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:21:46.980306512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:21:47.717338147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:21:47.724202927Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:21:49.092639149Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T14:21:50.176317492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:21:50.180360332Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:21:51.239631954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:21:51.24325784Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:21:51.769177434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:21:52.895727605Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-02-16T14:21:53.093323435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:21:53.096946792Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
ts=2026-02-16T14:21:53.861482281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49390"
level=info ts=2026-02-16T14:21:54.857378067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:21:54.86143265Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:21:56.698311707Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=debug ts=2026-02-16T14:21:56.698338656Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=17
level=warn ts=2026-02-16T14:21:56.768634189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:21:56.801750763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:21:56.805510539Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:21:57.061387448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:21:57.070018603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T14:21:57.330999945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:21:57.371233202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:21:57.375472374Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:22:00.501650406Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-02-16T14:22:00.573947185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:22:00.577779765Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:22:00.82253798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="44 kB" avg_comp="22 kB" total_uncomp="307 kB" avg_uncomp="154 kB" idle=1 full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:22:00.82601925Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:22:01.747875708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:22:01.748950404Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:22:01.768432862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:22:01.768461231Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:22:01.768470081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:22:01.768524576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:22:04.303633322Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=info ts=2026-02-16T14:22:04.731944136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:22:04.735615662Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:22:06.768688649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:22:08.106675172Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:22:08.178096164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:22:08.181717528Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:22:11.269249981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:22:11.272719618Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:22:11.307789644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:22:11.311617973Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:22:11.537627497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:22:11.541382083Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:22:11.769213763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:22:11.909771949Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=debug ts=2026-02-16T14:22:12.047061414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:22:12.048806829Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:22:13.234431357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:22:13.237896487Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:22:14.11931681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:22:14.123631Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:22:14.551605063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:22:14.554986165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:22:14.934362882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:22:14.937904772Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:22:15.711946311Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-02-16T14:22:16.769463426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:22:17.580012435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:22:17.584135275Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:22:18.241787356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:22:18.741167493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:22:18.746470892Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:22:19.515161421Z caller=checkpoint.go:378 msg="writing series" size="7.3 kB"
ts=2026-02-16T14:22:21.170013649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43166"
level=warn ts=2026-02-16T14:22:21.768627842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:22:22.706968027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="896 B" avg_comp="448 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:22:22.711048889Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:22:23.317914714Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=warn ts=2026-02-16T14:22:26.768661316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:22:27.1209116Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
ts=2026-02-16T14:22:27.33334766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:22:30.923719057Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-02-16T14:22:30.92374397Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-02-16T14:22:31.768442936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:22:31.76847065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:22:31.768478832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:22:31.768499357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:22:32.604782673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:22:32.611117871Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:22:32.754911638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:22:32.758871991Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:22:34.51165237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:22:34.514979478Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:22:34.686682264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:22:34.690498494Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:22:34.690519337Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:22:34.690542656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:22:34.693467336Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:22:34.726620266Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=warn ts=2026-02-16T14:22:36.769361245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:22:36.937846889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:22:36.94260532Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:22:37.236715367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:22:37.246023159Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:22:38.528914888Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T14:22:39.04783076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:22:39.053821162Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:22:40.06023473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:22:40.065837906Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:22:41.76851665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:22:42.332368161Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:22:42.359778865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:22:42.368378534Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:22:42.502575954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:22:42.506358446Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:22:42.522414916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:22:42.525922744Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:22:45.834122243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:22:45.837941495Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:22:45.837973187Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:22:45.838006059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:22:45.841574463Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:22:46.13561566Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-02-16T14:22:46.76920889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:22:49.939880739Z caller=checkpoint.go:378 msg="writing series" size="887 kB"
level=debug ts=2026-02-16T14:22:49.939915179Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-02-16T14:22:51.702665922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:22:51.768706982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:22:53.011000832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:22:53.014485366Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:22:53.772456107Z caller=checkpoint.go:378 msg="writing series" size="25 MB"
level=debug ts=2026-02-16T14:22:53.772482431Z caller=checkpoint.go:390 msg="flushing series" totalSize="25 MB" series=1
ts=2026-02-16T14:22:53.864682073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:54986"
level=info ts=2026-02-16T14:22:54.452175042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:22:54.456356238Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T14:22:55.886642857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:39698"
level=info ts=2026-02-16T14:22:56.765002265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:22:56.768503989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-02-16T14:22:56.769101387Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:22:56.978534227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:22:56.98338676Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:22:57.33459095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:22:57.542991404Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=info ts=2026-02-16T14:22:57.654265855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:22:57.658315914Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:23:01.346214309Z caller=checkpoint.go:378 msg="writing series" size="579 kB"
level=info ts=2026-02-16T14:23:01.541280984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:23:01.545014919Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:23:01.681453782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:23:01.685374119Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:23:01.748135395Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:23:01.748156961Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:23:01.768663835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:23:01.76870016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T14:23:01.768697411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:01.768709517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:23:02.325902419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:23:02.329417753Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:23:03.367941722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:23:03.540442049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:23:03.543646927Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:23:04.857766243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:23:04.861319519Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:23:04.901386873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="1.9 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:23:04.905050354Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:23:05.14904051Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:23:05.559820008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:23:05.563317121Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:23:06.263627595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:23:06.26714011Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:23:06.768503617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:07.297616653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:23:07.301639084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:23:08.819848831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:23:08.823532217Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:23:08.952132017Z caller=checkpoint.go:378 msg="writing series" size="202 kB"
level=warn ts=2026-02-16T14:23:11.768858357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:23:12.755058143Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T14:23:14.891306086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:23:14.895334119Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:23:16.31534284Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:23:16.557218338Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T14:23:16.769278669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:18.760376686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:23:18.764576937Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:23:18.866834497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="914 B" avg_comp="457 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:23:18.871471456Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:23:20.360185239Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T14:23:20.606889513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:23:20.611798931Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:23:21.175254636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:23:21.181756728Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:23:21.768646911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:23.049815286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:23:23.053767643Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:23:23.320668902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:23:23.325171203Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:23:23.982302065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:23:23.986333217Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:23:24.165042217Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-02-16T14:23:24.16506753Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=8
level=info ts=2026-02-16T14:23:24.965800259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:23:24.969199555Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:23:26.768530068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:23:27.3362088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:23:27.754343431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:23:27.756775355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:23:27.76028241Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:23:27.760288747Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:23:27.966293035Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=debug ts=2026-02-16T14:23:31.768222868Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.7 kB" series=1
level=info ts=2026-02-16T14:23:31.769290993Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:23:31.76931579Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:23:31.769324536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:23:31.769340523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:31.771886908Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-02-16T14:23:31.773120046Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.004961704s
level=info ts=2026-02-16T14:23:33.233488085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:23:33.238750936Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:23:36.768714304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:38.996699939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:23:39.003021253Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:23:40.330538552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:23:40.334771208Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:23:40.334801196Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:23:40.334832452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:23:40.338203949Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:23:41.768808444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:46.358447452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:23:46.362910567Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:23:46.768923535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:23:46.80746432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:23:47.747817661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:23:48.199017033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:23:48.202938265Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:23:49.491548527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:23:49.496467761Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:23:50.32575766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:23:50.329371226Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:23:50.727694441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:23:50.73198686Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:23:51.160604218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:23:51.164311075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:23:51.7690764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:23:51.929710864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:23:51.933891483Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:23:52.416542764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:23:52.42052316Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:23:53.319541545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:23:53.323452008Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:23:53.709973371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:23:53.713967098Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:23:53.867955668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:38598"
level=info ts=2026-02-16T14:23:55.000085053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:23:55.004259562Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T14:23:55.890003911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:55670"
level=warn ts=2026-02-16T14:23:56.769398673Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:23:57.337716363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:23:58.934064754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:23:58.937875491Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:23:59.550135376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:23:59.553759362Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:24:01.748083828Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:24:01.748256291Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:24:01.768198511Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:24:01.768657259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:24:01.768668638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:01.768637321Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-02-16T14:24:01.768682041Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:24:01.768690346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:24:01.768949463Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T14:24:02.503168279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="914 B" avg_comp="457 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:24:02.507047928Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:24:05.446985136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:24:05.451030054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:24:05.519179246Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-02-16T14:24:06.768795344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:08.819020013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:24:08.822918003Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:24:09.269294802Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-02-16T14:24:11.769447255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:11.842644016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:24:11.849142045Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:24:12.18427766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:24:12.18869798Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:24:13.019970147Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T14:24:13.020011902Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-02-16T14:24:13.553967315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:24:13.561682884Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:24:13.561707815Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:24:13.561737409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:24:13.568855288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:24:14.943985694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:24:14.948844843Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:24:15.772696474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:24:15.776565654Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:24:16.769630152Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=warn ts=2026-02-16T14:24:16.769637855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:24:20.519513641Z caller=checkpoint.go:378 msg="writing series" size="380 kB"
level=warn ts=2026-02-16T14:24:21.768956051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:21.772090919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:24:21.776361975Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:24:21.776382142Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:24:21.776423621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:24:21.7798792Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:24:22.405861858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:24:22.410335647Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:24:22.911120733Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:24:23.843165803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:24:23.847181216Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:24:24.269513408Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
ts=2026-02-16T14:24:25.891121509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:52104"
level=info ts=2026-02-16T14:24:26.369218484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:24:26.381147643Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:24:26.76954655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:24:27.339732196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:24:28.019243105Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:24:28.441216376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:24:28.447214452Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:24:28.447241276Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:24:28.447269854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:24:28.451801773Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:24:29.263056256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:24:29.267849841Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:24:31.769089257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:24:31.769131827Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:24:31.769153784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:24:31.76916407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:24:31.769249291Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-02-16T14:24:31.857665403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:24:31.861642085Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:24:32.571811786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:24:32.575874968Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:24:33.062211003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:24:33.066065582Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:24:33.302573342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:24:33.306769152Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:24:35.275161912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:24:35.279002613Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:24:35.519580033Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T14:24:36.769321982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:36.87863721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:24:36.88260736Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:24:38.730138637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:24:38.735047382Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:24:38.735080016Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:24:38.73511567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:24:38.739483663Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:24:38.831876403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:24:38.836422026Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:24:39.269454463Z caller=checkpoint.go:378 msg="writing series" size="570 kB"
level=debug ts=2026-02-16T14:24:39.269479592Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-02-16T14:24:39.34353021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:24:39.351058164Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:24:40.094174037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:24:40.099159823Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:24:40.751227656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:24:40.755590148Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:24:41.768611735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:42.881446501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:24:42.885852937Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:24:43.026625188Z caller=checkpoint.go:378 msg="writing series" size="4.7 MB"
level=debug ts=2026-02-16T14:24:43.026659908Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.7 MB" series=1
level=debug ts=2026-02-16T14:24:43.049162959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:24:45.131919426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:24:45.136913432Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:24:46.210015578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:24:46.215556028Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:24:46.768938418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:24:46.76893604Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T14:24:50.464824443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:24:50.468394803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:24:50.519551399Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T14:24:50.537825462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="914 B" avg_comp="457 B" total_uncomp="698 B" avg_uncomp="349 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:24:50.542200781Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T14:24:51.181641946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45082"
level=warn ts=2026-02-16T14:24:51.76920827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:24:52.900387203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:24:52.904541947Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:24:54.269216219Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T14:24:54.558149985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:24:54.562877659Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:24:54.828826564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:24:54.839920673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:24:54.957204433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:24:54.961281005Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:24:55.166225642Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:24:55.647545285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:24:55.652075148Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:24:56.768621193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:24:57.341057723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:24:58.019746511Z caller=checkpoint.go:378 msg="writing series" size="680 kB"
level=info ts=2026-02-16T14:24:58.396677051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:24:58.403715603Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:25:01.748071562Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:25:01.748077946Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:25:01.768715454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:25:01.768758467Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:25:01.768767471Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:25:01.768778316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:01.770078851Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:25:02.383267719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:25:02.387157751Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:25:05.519345381Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T14:25:06.769454321Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:07.527947528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:07.532208392Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:07.532239353Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:25:07.532272969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:07.536207113Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:25:09.26954416Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T14:25:09.498822261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:25:09.509968561Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:25:11.215022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:25:11.219141095Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:25:11.768870367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:12.182943817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:25:12.186773726Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:25:12.186794582Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:25:12.18681973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:25:12.189887961Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:25:13.019668391Z caller=checkpoint.go:378 msg="writing series" size="252 kB"
level=debug ts=2026-02-16T14:25:13.019698573Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-02-16T14:25:13.761090498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:25:13.764865477Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:25:15.143783979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:25:15.147496193Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:25:16.025933265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:25:16.030370445Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:25:16.769216133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:16.769232551Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-02-16T14:25:19.420052898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:25:19.424121745Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:25:19.632133239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:25:19.635955366Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:25:19.635981641Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:25:19.636008057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:25:19.639475805Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:25:20.522993812Z caller=checkpoint.go:378 msg="writing series" size="6.2 MB"
level=debug ts=2026-02-16T14:25:20.523026277Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.2 MB" series=2
ts=2026-02-16T14:25:21.183849449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41598"
level=info ts=2026-02-16T14:25:21.53328352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:21.537241882Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:25:21.768668531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:25:23.872368554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:60512"
level=debug ts=2026-02-16T14:25:24.269874373Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:25:24.538462353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:25:24.542569514Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:25:25.975311462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:25:25.978834501Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:25:26.082384508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:25:26.085668834Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:25:26.768961996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:26.923945045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:25:26.928839571Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T14:25:27.343674645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:25:27.950832576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:25:27.953965279Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:25:28.019740985Z caller=checkpoint.go:378 msg="writing series" size="995 kB"
level=debug ts=2026-02-16T14:25:28.117599582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:25:28.394156764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:25:29.911736734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:25:29.915249546Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:25:31.76872229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:25:31.768752258Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:25:31.768761672Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:25:31.768757226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:31.769853936Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:25:32.953325889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:25:32.957256743Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:25:35.519947965Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-02-16T14:25:35.8129389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:25:35.816881768Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:25:36.265119342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:25:36.268650182Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:25:36.769326379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:36.779450784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:25:36.78289097Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:25:36.782912917Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:25:36.782934761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="158 B" avg_comp="158 B" total_uncomp="128 kB" avg_uncomp="128 kB" idle=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:25:36.786303002Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:25:38.254516483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:25:38.258297666Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:25:39.269611472Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-02-16T14:25:39.26963814Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-02-16T14:25:39.445427732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:25:39.448746815Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:25:40.877054357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:25:40.88082482Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:25:41.768464025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:43.019569622Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=info ts=2026-02-16T14:25:44.714185794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:25:44.719996549Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:25:45.232669786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:25:45.236471097Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:25:46.415281448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" idle=1 full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:25:46.419154999Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:25:46.419179627Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:25:46.419209506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:25:46.422329311Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:25:46.768747805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:46.769941909Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-02-16T14:25:50.169215959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:25:50.172794827Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T14:25:50.519442706Z caller=checkpoint.go:378 msg="writing series" size="281 kB"
level=info ts=2026-02-16T14:25:50.920013415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:25:50.924248896Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:25:51.768952292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:52.162219423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:25:52.169785717Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:25:52.208861258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:52.212111975Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:25:53.225757771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:25:53.230017512Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:25:53.230050933Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:25:53.230084743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:25:53.233623342Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:25:53.284735711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:25:53.288141706Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:25:53.908093858Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:25:54.168369962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:25:54.171905089Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:25:54.269751355Z caller=checkpoint.go:378 msg="writing series" size="953 kB"
level=debug ts=2026-02-16T14:25:54.269786072Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-02-16T14:25:55.301663212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:25:55.305316171Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:25:56.769364779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:25:57.345791782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:25:58.019363897Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T14:25:58.885029362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:25:58.888710548Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:26:00.063819256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:26:00.067690977Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:26:00.58326587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:26:00.586815658Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:26:00.586843249Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:00.586873744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:26:00.589914767Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:26:01.748081163Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:26:01.748291731Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:26:01.768723965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:26:01.768760633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:26:01.768768337Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:26:01.768808857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:01.7699412Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-02-16T14:26:05.51978408Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T14:26:06.769128525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:08.855245668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:26:08.859607147Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:26:09.269885183Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T14:26:10.742057702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:26:10.74725536Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:26:11.768827972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:11.83711074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:26:11.84184149Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:26:11.841879619Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:11.841915494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:26:11.845227234Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:26:12.399422921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:26:12.403383929Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:26:12.403427168Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:12.403452534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:26:12.406571093Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:26:12.501857464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:12.506541598Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:12.506567305Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:12.703864995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:12.707393321Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:26:13.019772896Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=info ts=2026-02-16T14:26:14.24169297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:14.245166079Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:14.245189369Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:14.24521243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:14.248238327Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:26:15.737754633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:26:15.741484687Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:26:15.888774791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:26:15.892090682Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:26:16.769153781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:16.769168215Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T14:26:17.689532953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:26:17.692971609Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:26:17.692992534Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:17.693013842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:26:17.695839383Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:26:18.744521759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:26:18.747972086Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:26:18.747992716Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:18.748014304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:26:18.750864369Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:26:20.519714967Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
ts=2026-02-16T14:26:21.187751864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:38704"
level=warn ts=2026-02-16T14:26:21.769083101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:22.944680485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:26:23.708527078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:26:23.712144207Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:26:23.712166359Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:23.712187744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:26:23.716819083Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T14:26:23.87548402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55236"
level=info ts=2026-02-16T14:26:23.911068918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:26:23.914528231Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:26:24.26988558Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:26:24.80896582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:26:24.818852067Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:26:25.867898641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:26:25.871446427Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:26:26.700733993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:26:26.704656528Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T14:26:26.769460781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:26:27.347468207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:26:28.019867314Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-02-16T14:26:28.529370581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:28.533258399Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:26:28.72483578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:26:28.728974378Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:26:28.851631526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:26:28.854921268Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:26:29.265150704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:26:29.268682249Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:26:29.268704516Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:29.268730383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:26:29.271626036Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:26:30.361936395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:26:30.365690663Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:26:30.365719336Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:30.365748834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:26:30.36863768Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:26:30.377387694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:26:30.379057073Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:26:30.668535121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:30.67221631Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:26:30.798824102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:26:30.802360338Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:26:31.082154607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:26:31.085620464Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:26:31.76897176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:31.768966291Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:26:31.768989939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:26:31.769011446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:26:31.769021346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:26:32.645533546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:26:32.648947834Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:26:33.461584535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:26:33.46470874Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:26:35.518994961Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:26:36.20301499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:26:36.208651346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:26:36.768613663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:37.410737635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:26:37.417911049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:26:38.284088686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:26:38.288007132Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:26:39.269516368Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=info ts=2026-02-16T14:26:39.454414061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:26:39.457690152Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:26:41.12475936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:26:41.130813473Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:26:41.144187275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:26:41.14769526Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:26:41.492707771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:26:41.496305221Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:26:41.496331119Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:26:41.496354883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:26:41.499115986Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:26:41.769282554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:43.020019717Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=info ts=2026-02-16T14:26:45.13745582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:26:45.141571017Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:26:45.450726068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:26:45.454145733Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:26:45.969886221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:26:45.974053336Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:26:46.174582637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="40 kB" avg_comp="20 kB" total_uncomp="391 kB" avg_uncomp="196 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:26:46.178280691Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:26:46.769358536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:46.769432943Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:26:49.104454829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:26:49.108216128Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:26:50.519869011Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-02-16T14:26:51.190127214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60074"
level=warn ts=2026-02-16T14:26:51.769448698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:51.882695814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:26:51.888036098Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:26:52.337382144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:26:52.343224616Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:26:54.269925258Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T14:26:55.926567974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:26:55.930317711Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T14:26:56.314642938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:26:56.769469677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:26:57.349023623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:26:57.594617867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:57.606422773Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:26:57.655608032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:57.659499184Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:26:58.019571665Z caller=checkpoint.go:378 msg="writing series" size="605 kB"
level=debug ts=2026-02-16T14:26:58.019601504Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=17
level=info ts=2026-02-16T14:26:59.951980423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:26:59.955794341Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:27:01.747695883Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:27:01.748762453Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:27:01.768430119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:27:01.768457414Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:27:01.768465253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:27:01.768454674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:01.769627673Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=info ts=2026-02-16T14:27:02.21602338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:27:02.219703586Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:27:03.209659522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="97 kB" avg_uncomp="97 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:27:03.212942817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:27:05.320515566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:27:05.324449178Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:27:05.519250225Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=info ts=2026-02-16T14:27:06.545297698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:27:06.549026811Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:27:06.769321141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:07.48767237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:27:07.491706861Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:27:09.269836682Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:27:10.649858878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="3.4 kB" avg_comp="1.7 kB" total_uncomp="9.9 kB" avg_uncomp="5.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:27:10.653630779Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:27:11.197840279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:27:11.769452376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:13.020010058Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=info ts=2026-02-16T14:27:15.294718063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:27:15.298573328Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:27:16.768879226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:16.768999615Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T14:27:16.978456666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:27:16.982044031Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:27:18.207080263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:27:18.210754355Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:27:20.51073912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:27:20.514158328Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:27:20.519241752Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-02-16T14:27:21.769544919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:23.895825873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:27:23.90260195Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:27:23.90262622Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:27:23.902651451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:27:23.908546913Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:27:24.26983705Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-02-16T14:27:24.653786951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:27:26.769016985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:27:27.350977039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:27:28.019698862Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-02-16T14:27:28.766149724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:27:28.771444164Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:27:28.852691918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:27:28.85645387Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:27:28.966820856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:27:28.97013801Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:27:29.641281792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:27:29.644826406Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:27:30.01606377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:27:30.019929501Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:27:31.769047256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:27:31.76904229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:31.769075246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:27:31.769083759Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:27:31.769148421Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-02-16T14:27:32.898617765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" idle=1 full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:27:32.902516391Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:27:35.519965052Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=debug ts=2026-02-16T14:27:35.519995445Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-02-16T14:27:36.377520223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:27:36.385550949Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:27:36.769489202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:36.797636049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:27:36.806098381Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:27:39.269644189Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T14:27:40.235356658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:27:40.239037495Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:27:41.768903496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:42.750196465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:27:42.757360914Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:27:43.019079601Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:27:43.993592615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:27:43.997564691Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:27:44.192163562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:27:44.196076338Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:27:44.457686951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:27:44.461098889Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:27:45.457053498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:27:45.461283066Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:27:46.769363407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:46.769516431Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T14:27:49.800354834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:27:49.803940667Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:27:50.520274507Z caller=checkpoint.go:378 msg="writing series" size="984 kB"
level=debug ts=2026-02-16T14:27:50.520297455Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-02-16T14:27:51.76900447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:53.160784456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" idle=1 full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:27:53.165480725Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:27:53.259723931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:27:53.265622235Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:27:53.935452866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="1.9 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:27:53.940492734Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:27:54.306017627Z caller=checkpoint.go:378 msg="writing series" size="30 MB"
level=debug ts=2026-02-16T14:27:54.306052645Z caller=checkpoint.go:390 msg="flushing series" totalSize="30 MB" series=1
level=info ts=2026-02-16T14:27:54.694375698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:27:54.698068595Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:27:54.713275047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:27:56.769333481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:27:57.352524603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:27:58.019292691Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=info ts=2026-02-16T14:27:58.491628316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" idle=1 full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:27:58.495546365Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:27:59.628549828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" idle=1 full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:27:59.632014005Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:28:01.739315389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:28:01.743078487Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:28:01.747974228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:28:01.749059799Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:28:01.768523352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:28:01.768552724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:28:01.768561724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:28:01.768554324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:28:01.770014545Z caller=checkpoint.go:378 msg="writing series" size="573 kB"
level=debug ts=2026-02-16T14:28:05.519559757Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-02-16T14:28:05.597538516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:28:06.768884011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:08.894822249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:28:08.898908759Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:28:09.269104571Z caller=checkpoint.go:378 msg="writing series" size="202 kB"
level=warn ts=2026-02-16T14:28:11.769457995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:12.045087707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="340 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:28:12.048894965Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:28:12.841418135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:28:12.845238839Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:28:13.019728484Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T14:28:14.609922029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:28:14.613920615Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:28:16.769207443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:28:16.769233661Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T14:28:17.60953728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:28:17.615733555Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:28:18.425846215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:28:18.429943849Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:28:18.429975799Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:28:18.430010617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:28:18.433709691Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:28:18.744652393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:28:18.749770702Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:28:19.956187646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:28:19.960700807Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:28:20.519076639Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=debug ts=2026-02-16T14:28:20.993819273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:28:21.081986129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:28:21.085616913Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:28:21.76878169Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:22.348428679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:28:22.351547685Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:28:22.357623561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:28:22.360703037Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:28:24.272274055Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T14:28:24.272304337Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.6 MB" series=8
level=info ts=2026-02-16T14:28:26.658752352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:28:26.663125032Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:28:26.768686647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:28:27.353965905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:28:28.019946585Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T14:28:30.78452633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:28:30.788257831Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:28:31.229372983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:28:31.236481665Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:28:31.769132991Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.7 kB" series=1
level=info ts=2026-02-16T14:28:31.76914963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:28:31.76917346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:28:31.769182951Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:28:31.769191576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:31.773171561Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-02-16T14:28:31.774514911Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005632871s
level=info ts=2026-02-16T14:28:32.583220728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:28:32.586651912Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:28:33.820386857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:28:33.823893667Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:28:34.933846667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:28:34.937236372Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:28:35.028458661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:28:35.032463971Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:28:36.769175855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:38.000566786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:28:38.00429552Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:28:39.706241672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:28:39.712280302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:28:40.577982817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:28:40.58175904Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:28:41.337965685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:28:41.341384106Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:28:41.585021856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:28:41.588226537Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:28:41.768704805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:41.892008065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:28:41.895275253Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:28:42.143452814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:28:42.148004606Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:28:44.039242242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:28:44.043224349Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:28:46.707248905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:28:46.710855553Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:28:46.76914638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:28:51.199103714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:44734"
level=warn ts=2026-02-16T14:28:51.769361823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:28:55.906399526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:59310"
level=debug ts=2026-02-16T14:28:56.069680598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:28:56.638353334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:28:56.642190124Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:28:56.768821201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:28:57.355851625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:28:59.666156878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:28:59.671147633Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:29:01.747810859Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:29:01.748856427Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:29:01.768265723Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:29:01.768432338Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-02-16T14:29:01.768455958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:29:01.768482707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:29:01.768493027Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:29:01.768497405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:01.768680807Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-02-16T14:29:03.654309749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:29:04.387586153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:29:04.390913899Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:29:05.519538648Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=info ts=2026-02-16T14:29:05.878802658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:29:05.882475648Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:29:05.882501848Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:29:05.882533201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:29:05.885351883Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:29:06.768868092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:07.37895336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:29:07.382455268Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:29:07.661207772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:29:07.664671779Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:29:07.664692829Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:29:07.664715083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:29:07.667527265Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:29:08.834574352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:29:08.838721426Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:29:09.26950222Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-02-16T14:29:11.769185678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:12.183840856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:29:12.188243075Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:29:12.551475847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:29:12.55563629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:29:12.857202529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:29:12.860988847Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:29:13.020292153Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T14:29:13.020315999Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-02-16T14:29:14.809532113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:29:14.815701867Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:29:16.468388715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:29:16.471756219Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:29:16.768502184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:16.769615655Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:29:16.975145549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:29:16.980876751Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:29:18.963841839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:29:18.967540418Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:29:20.519830577Z caller=checkpoint.go:378 msg="writing series" size="477 kB"
level=warn ts=2026-02-16T14:29:21.768458091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:22.15693163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:29:22.160442222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:29:22.179461022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:29:22.183612308Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:29:24.269443699Z caller=checkpoint.go:378 msg="writing series" size="296 kB"
level=debug ts=2026-02-16T14:29:24.330646479Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:29:24.364822412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:29:24.368512967Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:29:24.490703124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:29:24.494251748Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:29:24.832431984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:29:24.836400012Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:29:25.349861148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:29:25.353962469Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T14:29:25.907961901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46612"
level=info ts=2026-02-16T14:29:26.595679822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:29:26.599994281Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=warn ts=2026-02-16T14:29:26.769322753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:29:27.358104563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:29:28.019567352Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:29:29.462808239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:29:29.471170432Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:29:29.49432025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:29:29.498079192Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:29:30.774013235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:29:30.778674547Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:29:31.768515689Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:29:31.768544249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:29:31.768554216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:29:31.768578086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:31.769754997Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
level=debug ts=2026-02-16T14:29:31.769772107Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-02-16T14:29:33.446616919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:29:33.450958483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:29:35.519562563Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T14:29:36.420590668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:29:36.424327221Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:29:36.768999448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:38.121924088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:29:38.126079775Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:29:38.413867659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:29:39.269161232Z caller=checkpoint.go:378 msg="writing series" size="570 kB"
level=warn ts=2026-02-16T14:29:41.768774743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:42.333062536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:29:42.338152355Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:29:42.724553426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:29:42.728587186Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:29:43.022728881Z caller=checkpoint.go:378 msg="writing series" size="6.0 MB"
level=debug ts=2026-02-16T14:29:43.022752542Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.6 MB" series=3
level=warn ts=2026-02-16T14:29:46.76928964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:46.769330213Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=info ts=2026-02-16T14:29:47.137821434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:29:47.142974129Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:29:49.350640006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:29:49.354686913Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:29:49.591477281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:29:49.595540248Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:29:49.690704667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:29:49.694047922Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:29:50.518949757Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=debug ts=2026-02-16T14:29:50.718640657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:29:51.768681832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:53.531621562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:29:53.535514359Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:29:54.268839655Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-02-16T14:29:55.909638547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:36504"
level=info ts=2026-02-16T14:29:56.144030607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:29:56.14994692Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:29:56.768508335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:56.863777375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:29:56.867564428Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T14:29:57.359819578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:29:57.784046878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:29:57.787877298Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:29:58.019273147Z caller=checkpoint.go:378 msg="writing series" size="609 kB"
level=info ts=2026-02-16T14:30:01.748026645Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:30:01.74842334Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=debug ts=2026-02-16T14:30:01.748666232Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T14:30:01.748714382Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968057 n=1
level=debug ts=2026-02-16T14:30:01.748758051Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968057 n=1
level=info ts=2026-02-16T14:30:01.768965772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:30:01.768994456Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T14:30:01.768990862Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:30:01.769003461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:30:01.76901749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:03.003580258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:30:03.007141158Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:30:03.922767049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:30:03.926850011Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:30:05.518944381Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T14:30:06.110207412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:30:06.11410776Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:30:06.769484923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:30:06.769512813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-02-16T14:30:06.773308134Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:30:06.77333674Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:30:06.773367392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:30:06.7765009Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:30:07.448545224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:30:07.452744581Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:30:07.73645601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:30:07.740355282Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:30:08.901441126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:30:08.905008705Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:30:09.269507762Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T14:30:10.457242524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:30:10.46126754Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T14:30:11.768713454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:12.236675595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:30:12.24032314Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:30:12.820107542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:30:12.828269737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:30:13.018904244Z caller=checkpoint.go:378 msg="writing series" size="252 kB"
level=debug ts=2026-02-16T14:30:13.018930465Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-02-16T14:30:16.769194312Z caller=checkpoint.go:378 msg="writing series" size="9.1 kB"
level=warn ts=2026-02-16T14:30:16.769235436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:20.282690988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:30:20.288835734Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:30:20.526742727Z caller=checkpoint.go:378 msg="writing series" size="6.7 MB"
level=debug ts=2026-02-16T14:30:20.526768776Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.7 MB" series=2
level=debug ts=2026-02-16T14:30:20.708245261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:30:21.571848875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:30:21.575305099Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:30:21.769352619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:22.423941021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:30:22.427758196Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:30:24.269656604Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:30:24.707131321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:30:24.710523823Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:30:25.792687045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:30:25.79603731Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:30:26.768953381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:30:27.360930745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:30:28.01993167Z caller=checkpoint.go:378 msg="writing series" size="944 kB"
level=info ts=2026-02-16T14:30:30.344290555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:30:30.348119699Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:30:30.364192462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:30:30.370023067Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:30:31.768732964Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:30:31.76873599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:30:31.76875284Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T14:30:31.768764687Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:30:31.768775365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:30:32.000911289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:30:32.002681463Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:30:32.804948286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:30:32.809421121Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:30:33.249691327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:30:33.253975389Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:30:35.518900771Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-02-16T14:30:36.602983121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:30:36.606563945Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:30:36.769132936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:38.438058591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="46 kB" avg_comp="23 kB" total_uncomp="307 kB" avg_uncomp="154 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:30:38.441993404Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:30:39.269550547Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-02-16T14:30:39.26957629Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-02-16T14:30:41.769157141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:42.287464095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:30:42.293094761Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:30:42.851844251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:30:42.855296292Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:30:43.019019972Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=info ts=2026-02-16T14:30:43.102137606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:30:43.105597595Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:30:44.806967606Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:30:46.48606468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:30:46.489853767Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:30:46.700457151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:30:46.703882229Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:30:46.768932013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:30:46.769092942Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-02-16T14:30:50.012861463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:30:50.016283873Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:30:50.519825206Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
level=debug ts=2026-02-16T14:30:50.519850109Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-02-16T14:30:51.769058113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:51.778197392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:30:51.781546728Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:30:52.801455439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:30:52.804811281Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:30:52.877036982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:30:52.880507467Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:30:54.014751515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:30:54.018167819Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T14:30:54.269671402Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T14:30:54.269695891Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-02-16T14:30:55.756311363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:30:55.759762113Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:30:55.822862142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:30:55.826351581Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T14:30:55.911924633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:54378"
level=warn ts=2026-02-16T14:30:56.768950471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:30:57.361845438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:30:58.01968089Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T14:30:59.104798982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:30:59.11248489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:30:59.11251092Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:30:59.112538348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:30:59.120642694Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:30:59.920840414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:30:59.924378713Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:31:00.31747179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:31:00.321173761Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:31:00.321202264Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:00.321230462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:31:00.324452031Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:31:01.748026689Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:31:01.748050478Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=warn ts=2026-02-16T14:31:01.768697453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:01.768694271Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:31:01.768722091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:31:01.768730511Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:31:01.768749839Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=info ts=2026-02-16T14:31:03.42263932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:31:03.427468895Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:31:04.660818588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:31:04.664278632Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:31:05.519106512Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=warn ts=2026-02-16T14:31:06.769170551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:08.46637235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:31:08.470739088Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:31:09.269273308Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T14:31:09.348912468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:31:09.356959773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:31:10.498462747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:10.502302626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:10.502330241Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:10.502357492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:10.505708452Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:31:11.668676952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:31:11.768833502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:12.584235084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:31:12.587653562Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:31:12.752358707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:31:12.755910247Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:31:13.019839508Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=info ts=2026-02-16T14:31:13.514051764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:31:13.517297437Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:31:16.136801943Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:31:16.769133469Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T14:31:16.76914385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:20.518860074Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:31:21.129388381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:31:21.133459302Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:31:21.76845588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:23.566447502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:31:23.570235983Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:31:23.894215609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:54404"
level=debug ts=2026-02-16T14:31:24.269425346Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:31:24.467849857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:24.472460721Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:31:25.897832143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:31:25.902720535Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:31:25.902744164Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:25.902768734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:31:25.906282811Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T14:31:25.913580815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:55268"
level=warn ts=2026-02-16T14:31:26.768868451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:31:27.362878406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:31:27.458004386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:31:27.461904629Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:31:28.018666267Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-02-16T14:31:28.264532846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="346 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:31:28.267718978Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:31:28.742050878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="310 kB" avg_uncomp="103 kB" idle=1 full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:31:28.745665925Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:31:29.748732898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:31:29.755118825Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:31:31.768386371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:31:31.768431904Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:31:31.76844056Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:31:31.76843249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:31.769535893Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:31:35.308540783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:31:35.312550268Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:31:35.518689995Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:31:35.858901849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:31:35.862602757Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T14:31:36.768654221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:38.134597546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:31:38.138431164Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:31:38.138460197Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:38.138490358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:31:38.141888776Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:31:39.268914663Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=info ts=2026-02-16T14:31:39.948017275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:31:39.952370839Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:31:40.967325077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:31:40.971758844Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:31:41.768996647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:41.909425755Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:31:42.168078826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:31:42.172052849Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:31:43.01953839Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=info ts=2026-02-16T14:31:45.007204466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:31:45.010753719Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:31:45.905176561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:31:45.908541493Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:31:45.989923545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:31:45.993442117Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:31:45.993465622Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:45.993498984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:31:45.996193324Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:31:46.052386105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:31:46.055661579Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:31:46.055684952Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:46.294235525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:31:46.297530744Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:31:46.696641029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:31:46.705022832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:31:46.769279031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:46.769314925Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:31:47.872527665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:31:47.876028663Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:31:48.358524023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:48.36247526Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:31:49.32067346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:31:49.324154539Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:31:50.133702345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:31:50.137651179Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:31:50.440717333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:31:50.444420168Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:31:50.519485658Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T14:31:51.768898285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:31:53.895715568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:56470"
level=debug ts=2026-02-16T14:31:54.269398922Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T14:31:56.115832324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:31:56.119462026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:31:56.768916113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:56.798031838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:31:56.801572568Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T14:31:57.364688078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:31:58.019538828Z caller=checkpoint.go:378 msg="writing series" size="571 kB"
level=debug ts=2026-02-16T14:31:58.019561102Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=17
level=info ts=2026-02-16T14:31:58.456645832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:31:58.46025053Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:31:58.884567105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:31:58.888470162Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:32:00.61178978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:00.615549687Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:32:01.747777555Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:32:01.748851884Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:32:01.7693284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:32:01.769339934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:32:01.769361896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:32:01.769372636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:32:01.769473899Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-02-16T14:32:02.831829918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="2.0 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:32:02.836003182Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:32:04.62237352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:32:04.625824932Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:32:04.625849076Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:04.62587367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:32:04.628727156Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:32:05.518935904Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-02-16T14:32:06.769475712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:06.913848063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:32:06.917078724Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:32:08.055432974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:32:09.269062274Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:32:09.857704048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:32:09.863581554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:32:11.337759865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:32:11.768941849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:12.680019351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:32:12.68364062Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:32:13.018898525Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=info ts=2026-02-16T14:32:13.096999077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:32:13.100303002Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:32:16.768731999Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=warn ts=2026-02-16T14:32:16.768771973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:19.315549681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="442 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:32:19.319532466Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:32:19.31956278Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:19.319598897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="448 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:32:19.322702171Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:32:20.519293035Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T14:32:21.219327514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41300"
level=warn ts=2026-02-16T14:32:21.769238195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:21.882519023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:32:21.88759992Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:32:21.933655556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:32:21.936803061Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:32:22.459132609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:32:22.462612211Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:32:22.462633164Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:22.462656878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:32:22.465694322Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:32:22.567009019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:22.570367441Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:22.57039281Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:22.64059588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="46 kB" avg_comp="15 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:32:22.644046953Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:32:22.763606489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:22.767058521Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:32:24.26928144Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-02-16T14:32:26.768615553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:32:27.366566368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:32:28.019627725Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T14:32:28.232817684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:32:28.237201855Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:32:28.311309083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:32:28.314584851Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:32:28.314605974Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:28.314629441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:32:28.31736133Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:32:29.707475943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:32:29.71228819Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:32:30.079846843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:30.084095225Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:32:30.275074157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:32:30.278871071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:32:30.591843888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:32:30.595281425Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:32:31.769206899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:32:31.769235919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T14:32:31.769223831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:31.769244521Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:32:31.769357911Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-02-16T14:32:31.78643487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:31.789837714Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:31.789860992Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:31.78988465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:31.792665764Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:32:33.115825311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:32:33.119039453Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:32:33.996011756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="448 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:32:33.999635196Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:32:35.204353887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:32:35.207798562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:32:35.250013178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:32:35.253303537Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:32:35.253324574Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:35.253345471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:32:35.256036045Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:32:35.51881369Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=debug ts=2026-02-16T14:32:35.518836901Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-02-16T14:32:36.768526909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:37.599168504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:32:37.602747975Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:32:39.26894376Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T14:32:41.255624464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:32:41.259201548Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:32:41.768804201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:42.006319202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:42.009900493Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:32:42.075147531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:32:42.078808888Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:32:42.587467387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:42.590754464Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:32:42.975148582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:32:43.019261683Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:32:43.488645018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:32:43.496793146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:32:46.769177079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:32:46.769343239Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T14:32:49.267870417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:32:49.27166594Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:32:49.271695645Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:49.271727524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:32:49.274853269Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:32:50.31315299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:32:50.316810988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:32:50.520154393Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:32:50.520180456Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-02-16T14:32:51.768925329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:51.882267334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:32:51.885903987Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:32:52.435697644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:32:52.440326223Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:32:54.234322646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:32:54.237648909Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:32:54.288480439Z caller=checkpoint.go:378 msg="writing series" size="31 MB"
level=debug ts=2026-02-16T14:32:54.288508189Z caller=checkpoint.go:390 msg="flushing series" totalSize="31 MB" series=1
ts=2026-02-16T14:32:55.918070591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:45106"
level=warn ts=2026-02-16T14:32:56.768845723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:32:57.367345757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:32:58.018722251Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=debug ts=2026-02-16T14:32:58.516111728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:33:00.288154292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:33:00.2914568Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:33:00.541183655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="3.9 kB" avg_comp="1.9 kB" total_uncomp="10 kB" avg_uncomp="5.2 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:33:00.544541852Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66bb381e%3A19c66bb565a%3A5c50c6db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:33:00.54456803Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:00.544600652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:33:00.549281995Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:33:01.748031541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:33:01.748296112Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:33:01.769212569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:33:01.769239228Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:33:01.76924782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:33:01.769263968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:01.769602868Z caller=checkpoint.go:378 msg="writing series" size="715 kB"
level=info ts=2026-02-16T14:33:02.221449235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:33:02.224722769Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:33:02.224743187Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:02.224765826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:33:02.227313419Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:33:05.190180602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:33:05.193998269Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:33:05.468777283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:33:05.472480706Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:33:05.472501322Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:05.472523835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:33:05.475475825Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:33:05.519603Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:33:06.40886743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:33:06.414701098Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:33:06.768949247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:33:07.373916692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:33:07.3773934Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:33:08.779153184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:33:08.782544376Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:33:09.268891461Z caller=checkpoint.go:378 msg="writing series" size="202 kB"
level=debug ts=2026-02-16T14:33:09.268918374Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-02-16T14:33:11.40681292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:33:11.410392824Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:33:11.769197709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:13.019259554Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T14:33:13.295035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="448 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:33:13.298629814Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:33:14.402448651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:33:14.406951645Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T14:33:15.870242934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:33:16.768457531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:16.769556533Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T14:33:18.855539525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:33:18.859865497Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:33:19.859832726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:33:19.863487533Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:33:20.182330919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:33:20.185834176Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:33:20.265087684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:33:20.268600629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:33:20.519231756Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T14:33:21.16233202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:33:21.166277271Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T14:33:21.223214158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46888"
level=info ts=2026-02-16T14:33:21.710161517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:33:21.713506556Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:33:21.768806798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:33:22.169376723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:33:22.172617545Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:33:22.174670592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.2 kB" avg_comp="4.2 kB" total_uncomp="16 kB" avg_uncomp="16 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:33:22.177471142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:33:22.177489186Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:22.177509602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:33:22.180188579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:33:22.714549318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:33:22.719216355Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:33:22.719244194Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:22.719268564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:33:22.722273537Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:33:23.217936646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:33:23.221489935Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:33:23.61058607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:33:23.613906814Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:33:24.270244956Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-02-16T14:33:24.270268978Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=4
level=warn ts=2026-02-16T14:33:26.768549777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:33:27.369189324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:33:28.019488524Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:33:28.125817776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:33:28.129264215Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:33:28.18166804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:33:28.185044941Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:33:28.346592298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:33:28.350536218Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:33:31.769200692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:33:31.769229985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:33:31.769238326Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:33:31.769231848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:31.769231325Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.3 kB" series=1
level=info ts=2026-02-16T14:33:31.772297491Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-02-16T14:33:31.773506257Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.004924897s
level=info ts=2026-02-16T14:33:35.892550149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:33:35.896286462Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:33:36.76854904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:33:38.337256728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:33:38.341267083Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:33:39.459430027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:33:39.463317389Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:33:40.197200053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:33:40.200633081Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:33:40.327907446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:33:40.335104955Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:33:41.234956559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:33:41.238365367Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:33:41.768456428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:42.107133145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:33:46.768748414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:33:48.422229102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:33:48.42591922Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:33:48.425948319Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:48.425978478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:33:48.429077827Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:33:48.498449314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:33:50.542530583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:33:50.545907856Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:33:50.757918594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:33:50.761550213Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:33:51.7685388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:33:52.309093468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:33:52.313465025Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:33:53.789884338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:33:53.793345458Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
ts=2026-02-16T14:33:53.901386899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43864"
level=info ts=2026-02-16T14:33:54.997616169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:33:55.001013179Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T14:33:55.920663793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:43294"
level=info ts=2026-02-16T14:33:56.433414469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:33:56.436772571Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:33:56.768620576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:33:57.370482268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:33:58.011670335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:33:58.017128543Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:33:58.01715068Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:33:58.017175054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:33:58.022137016Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:33:58.505454728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:33:58.509430419Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:34:00.273551316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:34:00.277179994Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:34:01.747984226Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:34:01.749041256Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:34:01.768512829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:34:01.76854362Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:34:01.768541084Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T14:34:01.768543253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:01.76855457Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:34:01.768930937Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-02-16T14:34:01.769188815Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=info ts=2026-02-16T14:34:03.267365241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:34:03.27097687Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:34:03.759463108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:34:03.762723227Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:34:04.989468784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="448 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:34:04.992991404Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:34:05.000991716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:34:05.004444322Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:34:05.519605291Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-02-16T14:34:06.769347339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:08.170875431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:34:08.174456416Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:34:09.269190171Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T14:34:09.893050712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="464 kB" avg_comp="232 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:34:09.897780055Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:34:10.442467266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:34:10.446241269Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:34:11.76871104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:12.955989336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:34:12.959340736Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:34:13.020196732Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T14:34:13.020220914Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-02-16T14:34:13.546136854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:34:13.550039627Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:34:14.036360339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:34:14.04181975Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:34:15.89882818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:34:16.769094765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:34:16.769164091Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:34:17.446444799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:34:17.450034042Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:34:17.631468973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:34:17.635038072Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:34:19.733254508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:34:19.737012362Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:34:20.519958259Z caller=checkpoint.go:378 msg="writing series" size="376 kB"
level=info ts=2026-02-16T14:34:21.716790007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:34:21.720671608Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:34:21.76915799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:23.567767109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="435 kB" avg_comp="218 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:34:23.573308496Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:34:23.785833909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:34:23.787511835Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:34:24.269936291Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=info ts=2026-02-16T14:34:24.673146085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:34:24.67694744Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T14:34:25.922522325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:52806"
level=warn ts=2026-02-16T14:34:26.768589334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:34:27.372708323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:34:28.019195822Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:34:31.728454976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:34:31.732571364Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:34:31.768827117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:34:31.768855613Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:34:31.76886421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:34:31.768882161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:34:31.770020528Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=info ts=2026-02-16T14:34:32.419291785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:34:32.426013483Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:34:32.931492003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:34:32.945080643Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:34:33.441385186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:34:33.44646098Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:34:33.703120093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:34:33.706871076Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:34:35.519343865Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-02-16T14:34:36.769110612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:38.677827459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:38.681489902Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:38.681511957Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:34:38.681535728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:38.684431937Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:34:39.269710309Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=debug ts=2026-02-16T14:34:39.269736202Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-02-16T14:34:40.773620641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:34:40.777115889Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:34:41.736842521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:34:41.740431604Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:34:41.768775045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:34:43.023720255Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T14:34:43.023750043Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=1
level=info ts=2026-02-16T14:34:43.591723911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:34:43.595019238Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:34:44.165825047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:34:44.169378739Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:34:45.713934316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:34:45.738154605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.3 kB" avg_comp="448 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:34:45.741763844Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:34:46.76861009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:34:46.769695229Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=debug ts=2026-02-16T14:34:50.519473442Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T14:34:50.916866112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:34:50.920461083Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:34:51.243155018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:34:51.247294939Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:34:51.768740789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:51.820851374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:34:51.824123151Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:34:53.845999268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:34:53.851933816Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:34:54.269879537Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T14:34:54.574513677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:34:54.578451825Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:34:56.769382593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:34:57.374495838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=info ts=2026-02-16T14:34:57.564492752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:57.566304914Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:34:58.020212563Z caller=checkpoint.go:378 msg="writing series" size="849 kB"
level=debug ts=2026-02-16T14:34:59.47837035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:35:01.646618591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:35:01.650119897Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:35:01.748010113Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:35:01.748044245Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:35:01.768577243Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:35:01.768607507Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:35:01.768615788Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:35:01.76860785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:01.769719616Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:35:05.23419267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:35:05.23859059Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:35:05.519181388Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T14:35:06.768671278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:06.85592677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:35:06.859559115Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:35:07.863534443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="470 kB" avg_comp="235 kB" total_uncomp="4.8 MB" avg_uncomp="2.4 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:35:07.867436796Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:35:08.234399605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:35:08.238975445Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:35:09.117267248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:35:09.120630072Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:35:09.27025756Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=debug ts=2026-02-16T14:35:09.27028204Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-02-16T14:35:09.755767794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:35:11.404335973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:35:11.407960861Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:35:11.769130054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:13.020294269Z caller=checkpoint.go:378 msg="writing series" size="252 kB"
level=info ts=2026-02-16T14:35:14.182766554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="435 kB" avg_comp="218 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:35:14.18628231Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:35:15.523096563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:35:15.530279996Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:35:16.768899229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:16.770012987Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T14:35:17.341451464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:35:17.345145972Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:35:17.473463463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:35:17.478945329Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:35:18.966452235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:35:18.970108872Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:35:20.52405126Z caller=checkpoint.go:378 msg="writing series" size="7.5 MB"
level=debug ts=2026-02-16T14:35:20.524076995Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.8 MB" series=3
ts=2026-02-16T14:35:21.23107553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33354"
level=warn ts=2026-02-16T14:35:21.769044091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:23.545340119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:35:23.549066057Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:35:23.602156936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:35:23.605492777Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:35:23.765151631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:35:23.768624642Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T14:35:23.905715611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:52020"
level=debug ts=2026-02-16T14:35:24.269338681Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:35:24.439685909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:35:24.442998044Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:35:24.443019931Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:35:24.443042659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:35:24.445861572Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T14:35:25.926431134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:55342"
level=info ts=2026-02-16T14:35:26.295067916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:35:26.298382421Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:35:26.769376995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:35:27.376434718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:35:27.552769383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:35:27.556107062Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:35:28.020387431Z caller=checkpoint.go:378 msg="writing series" size="957 kB"
level=info ts=2026-02-16T14:35:31.768457304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:35:31.768485409Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:35:31.768493781Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:35:31.768510168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:31.769599Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:35:33.948576366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:35:33.955337959Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:35:34.877097415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" idle=1 full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:35:34.881070543Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:35:35.519774156Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=warn ts=2026-02-16T14:35:36.768487967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:36.769558634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:35:36.774766614Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:35:38.703857434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:35:38.707796664Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:35:38.998936013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:35:39.003365625Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:35:39.068739826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:35:39.27020861Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-02-16T14:35:39.270232076Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-02-16T14:35:40.00233288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:35:40.006066674Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:35:40.235771268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:35:40.239674917Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:35:41.768531839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:43.019754967Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=info ts=2026-02-16T14:35:43.10439638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:35:43.108624887Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:35:46.769253504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:46.769346263Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-02-16T14:35:50.519793994Z caller=checkpoint.go:378 msg="writing series" size="309 kB"
level=warn ts=2026-02-16T14:35:51.768754666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:53.775304003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:35:53.784809935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:35:54.154892701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:35:54.158779567Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:35:54.271234871Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:35:54.271257023Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=4
level=info ts=2026-02-16T14:35:55.197042084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:35:55.20115752Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:35:56.768700332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:56.995438992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="435 kB" avg_comp="218 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:35:56.998805635Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
ts=2026-02-16T14:35:57.379092912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:35:57.640193964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:35:57.643660356Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:35:57.8779551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" idle=1 full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:35:57.882320979Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:35:57.955051944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:35:58.02022946Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T14:35:59.502310451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:35:59.506018559Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:36:00.85981032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:36:00.863140729Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:36:01.747949228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:36:01.748998678Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:36:01.768985697Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:36:01.769010896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:36:01.769018848Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:36:01.769032998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:01.770188055Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=info ts=2026-02-16T14:36:02.888795154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:36:02.893179521Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:36:04.022458375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:36:04.026922162Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:36:05.519978958Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T14:36:06.349763015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:36:06.353319634Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:36:06.353339442Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:36:06.353360229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:36:06.356244866Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:36:06.769439915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:08.642800821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:36:08.646499759Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:36:09.269529111Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=debug ts=2026-02-16T14:36:11.148626022Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:36:11.769167364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:13.019615493Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-02-16T14:36:14.368310488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:36:14.375302011Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:36:14.623569378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="47 kB" avg_comp="16 kB" total_uncomp="310 kB" avg_uncomp="103 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:36:14.62717445Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:36:15.233819342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:36:15.237593468Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:36:15.757092146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:36:15.760590746Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:36:16.518922533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:36:16.522621594Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:36:16.768673141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:16.76979535Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T14:36:17.702719905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:36:17.7060391Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:36:20.519471165Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-02-16T14:36:21.768904783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:22.494443125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:36:22.497911517Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:36:22.752955027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:36:22.756457843Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:36:23.77146487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:36:23.774929874Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:36:24.242986029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:36:24.246349174Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:36:24.246371989Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:36:24.246396392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:36:24.249188035Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:36:24.269318791Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:36:24.880341644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:36:24.88413024Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:36:26.768751563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:36:27.381157286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:36:27.80245067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:36:27.805848814Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:36:28.020094195Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-02-16T14:36:30.443440994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:36:30.447562256Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:36:30.776665957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:36:30.780238609Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:36:31.768750434Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:36:31.768786228Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:36:31.768796431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:36:31.768812236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:31.769886356Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:36:32.122635035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:36:32.126317494Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:36:34.834467617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:36:34.839886989Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:36:35.519596418Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:36:36.757751101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:36:36.763148106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:36:36.769256361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:37.506469006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:36:37.506474959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="104 kB" avg_comp="104 kB" total_uncomp="665 kB" avg_uncomp="665 kB" idle=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:36:37.517122358Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=debug ts=2026-02-16T14:36:39.270166498Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-02-16T14:36:41.258448965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:36:41.26215468Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:36:41.768709473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:42.927743817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:36:42.931829606Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:36:43.019243497Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=info ts=2026-02-16T14:36:43.378342402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:36:43.38193387Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:36:44.754930404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="435 kB" avg_comp="218 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:36:44.758635119Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:36:46.768900816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:46.769923882Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:36:47.28445522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:36:47.2887606Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:36:47.69422048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:36:47.698337392Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:36:48.227890577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:36:48.232631004Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T14:36:50.353680912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:36:50.519925219Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T14:36:50.77212569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:36:50.775738173Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
ts=2026-02-16T14:36:51.237825664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60614"
level=warn ts=2026-02-16T14:36:51.768738586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:51.881088177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:36:51.884500631Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:36:54.154975048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:36:54.158515695Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:36:54.269821038Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-02-16T14:36:54.879731457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="47 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" idle=1 full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:36:54.883174917Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:36:56.769038286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:36:57.38269304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:36:57.912170176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:36:57.91554474Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:36:58.020218049Z caller=checkpoint.go:378 msg="writing series" size="608 kB"
level=debug ts=2026-02-16T14:36:58.020241838Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=17
level=info ts=2026-02-16T14:37:01.748149235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:37:01.748218721Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=warn ts=2026-02-16T14:37:01.768949414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:37:01.768952886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:37:01.768986772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:37:01.768995151Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:37:01.770095135Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-02-16T14:37:02.420558686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:37:02.424433333Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:37:04.33331721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:37:04.460715934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:37:04.464252239Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:37:05.519583825Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=warn ts=2026-02-16T14:37:06.768863426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:37:07.417868834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:37:07.423843774Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:37:08.079695946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:37:08.083777267Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:37:09.269702214Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:37:10.856325512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:37:10.859998432Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:37:11.69643052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:37:11.703352959Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:37:11.768976617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:37:13.019623706Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=info ts=2026-02-16T14:37:13.080684923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:37:13.084030611Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:37:13.122181561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:37:13.125460961Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:37:13.670907734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:37:13.674530652Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:37:15.963970222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:37:15.967521392Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:37:16.768687466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:37:16.769788487Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T14:37:19.169527288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="40 kB" avg_comp="14 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:37:19.172932849Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:37:20.520116792Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:37:21.051453472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:37:21.055316545Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:37:21.768462767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:37:23.808348466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:37:23.810322028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:37:23.811977977Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:37:23.811983023Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:37:23.812001336Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:37:23.812020426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:37:23.815233418Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:37:24.269547707Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=info ts=2026-02-16T14:37:24.745537544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:37:24.749190367Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:37:26.769445113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:37:27.231683802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:37:27.235315621Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
ts=2026-02-16T14:37:27.384057187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:37:28.019682793Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:37:28.66946858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:37:28.673645256Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:37:29.511918045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:37:29.51529241Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:37:31.768712605Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:37:31.768708385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:37:31.768736626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:37:31.768745092Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:37:31.769913702Z caller=checkpoint.go:378 msg="writing series" size="141 kB"
level=debug ts=2026-02-16T14:37:31.769926479Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-02-16T14:37:32.770733424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:37:32.775161442Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:37:32.775186532Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:37:32.775212388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:37:32.778955422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:37:32.907234923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:37:32.910964458Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:37:33.202838428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:37:33.206478243Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:37:33.296698385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:37:33.303945185Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:37:33.599972845Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:37:34.18438107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:37:34.188291509Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:37:35.519590785Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=info ts=2026-02-16T14:37:36.370368878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:37:36.374257636Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:37:36.76944108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:37:39.269224256Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T14:37:40.822476903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:37:40.831777443Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:37:40.915088736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:37:40.918672467Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:37:41.76937787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:37:43.019667436Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:37:43.188280918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:37:43.191958603Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:37:43.434805765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="435 kB" avg_comp="218 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:37:43.438496401Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:37:46.768939161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:37:46.770094437Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T14:37:49.825834343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:37:49.830273683Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:37:50.218845763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:37:50.222382341Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:37:50.520122062Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:37:50.520145129Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=5
level=debug ts=2026-02-16T14:37:51.430516007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:37:51.768741533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:37:53.898835871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:37:53.902638521Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:37:53.914737363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57542"
level=debug ts=2026-02-16T14:37:54.287955632Z caller=checkpoint.go:378 msg="writing series" size="32 MB"
level=debug ts=2026-02-16T14:37:54.287982061Z caller=checkpoint.go:390 msg="flushing series" totalSize="32 MB" series=1
level=info ts=2026-02-16T14:37:54.355943035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:37:54.360088759Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:37:56.768940551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:37:57.135846306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:37:57.142957816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:37:57.142983529Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:37:57.143011663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:37:57.149641761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T14:37:57.386092817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:37:57.6096339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:37:57.613054637Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:37:58.019883637Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-02-16T14:37:58.481219791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:37:58.484560193Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:37:59.183307466Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:38:00.939827667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="363 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:38:00.943308405Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:38:01.747504146Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:38:01.748557357Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:38:01.768983036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:38:01.769011213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:38:01.769019407Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:38:01.769031669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:38:01.769348956Z caller=checkpoint.go:378 msg="writing series" size="499 kB"
level=info ts=2026-02-16T14:38:02.967894991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:38:02.971351511Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:38:05.520108149Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-02-16T14:38:06.768751196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:06.770900407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:38:06.774261389Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:38:07.892179681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:38:07.895660845Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:38:09.269914624Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=info ts=2026-02-16T14:38:09.60753113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:38:09.614627431Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:38:11.769387643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:38:13.019503188Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T14:38:14.728349441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="57 kB" avg_comp="19 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:38:14.733309773Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:38:15.85930562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:38:15.863369737Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:38:16.769179694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:38:16.769181185Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T14:38:17.14368221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:38:17.146989875Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:38:18.817163641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:38:18.820922097Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:38:19.965617809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:38:19.969264248Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:38:20.463667759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:38:20.467038722Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:38:20.520213109Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
ts=2026-02-16T14:38:21.243161364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50486"
level=warn ts=2026-02-16T14:38:21.768802465Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:23.071113037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:38:23.074529983Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:38:23.197943368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:38:23.202046241Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=debug ts=2026-02-16T14:38:24.271166438Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-02-16T14:38:24.2711909Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=8
level=info ts=2026-02-16T14:38:24.604670956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:38:24.608616927Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:38:24.60864158Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:24.608667138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:38:24.611838311Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:38:26.769502526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:38:27.387955215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:38:27.438761916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:38:27.443592831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:38:28.019886852Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:38:28.181465264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:38:28.185274497Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:38:29.469219909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:38:29.472811199Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:38:29.472834467Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:29.472862473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:38:29.475938467Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:38:29.739288544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:38:29.746707199Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:38:29.952525343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:38:29.958560164Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:38:30.428787161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:38:30.432748071Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:38:30.432767547Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:30.432788547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:38:30.435910032Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:38:31.769169021Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.3 kB" series=1
level=info ts=2026-02-16T14:38:31.769181699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:38:31.769209155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:38:31.769219534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:38:31.769222044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:31.778284359Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-02-16T14:38:31.779394183Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010309611s
level=info ts=2026-02-16T14:38:31.984118237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:38:31.989525088Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:38:32.622711467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:38:32.627677772Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:38:33.8174662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:38:33.820885423Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:38:34.509610741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:38:35.232877054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:38:35.236696086Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:38:36.768957601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:36.937455378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:38:36.940844082Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:38:36.940868185Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:36.940890603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:38:36.943688427Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:38:40.524089428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:38:40.527517423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:38:41.769172075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:43.874193606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:38:43.877698925Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:38:44.357357766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:38:44.359327437Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T14:38:46.769436326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:47.511597249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:38:47.516995268Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:38:47.517020548Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:47.517045165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:38:47.519937441Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:38:48.478431885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:38:48.482027542Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:38:48.910262219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:38:48.913977767Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:38:49.699426248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:38:49.702923013Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:38:50.646457447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:38:50.649963771Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:38:51.76912587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:52.060989723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:38:52.064316378Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:38:52.064336492Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:52.064357768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:38:52.067265137Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T14:38:53.917666548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:41722"
ts=2026-02-16T14:38:55.938590314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:44792"
level=info ts=2026-02-16T14:38:56.505511621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:38:56.509015769Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:38:56.769027853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:38:56.880315072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:38:56.884471544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:38:56.884492584Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:56.884515431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:38:56.887278319Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
ts=2026-02-16T14:38:57.389475514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:38:57.720223398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:38:57.723721674Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:38:57.723743279Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:57.723767708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:38:57.726662335Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:38:59.549308312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="57 kB" avg_comp="19 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:38:59.552883827Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:38:59.552906181Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:38:59.552929761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="57 kB" avg_comp="19 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:38:59.555924879Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:39:00.756301042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:39:00.760241308Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:39:01.748041529Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:39:01.748232207Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:39:01.768793209Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T14:39:01.76881016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:01.76881385Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:39:01.768832925Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:39:01.768840761Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:39:01.768931873Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-02-16T14:39:01.769150478Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=info ts=2026-02-16T14:39:03.078622289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:39:03.082166242Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:39:03.788348884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:39:03.791775538Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:39:04.285551206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:39:04.28900181Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:39:04.289023478Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:04.589363794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:39:04.592920616Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:39:05.082035272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:39:05.08568464Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:39:05.085707553Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:05.085736723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:39:05.088760856Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:39:05.519975064Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=info ts=2026-02-16T14:39:06.084873555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:39:06.088110349Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:39:06.769090066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:07.024674564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:39:07.02814616Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:39:07.333522417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:39:07.336964462Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:39:07.934307646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:39:09.2691767Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T14:39:09.885747288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:39:09.889258394Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:39:09.889280481Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:09.889305149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:39:09.892163917Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:39:10.085706087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:39:10.089101822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:39:10.197588607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:39:10.201586969Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:39:11.512201269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:39:11.515569833Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:39:11.768690209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:12.554801962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="57 kB" avg_comp="19 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:39:12.558338276Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:39:13.020713881Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-02-16T14:39:13.020738932Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-02-16T14:39:13.358789505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="156 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:39:13.362736326Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:39:13.460022786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:39:13.467055029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:39:13.827171906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:39:13.833230958Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:39:15.185470705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:39:15.189035866Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:39:16.769101909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:39:16.769135871Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:39:17.016623088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:39:17.019977236Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:39:17.495214828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:39:17.498617283Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:39:18.244918782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:39:18.250445815Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:39:20.178672421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:39:20.182222384Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:39:20.520314981Z caller=checkpoint.go:378 msg="writing series" size="489 kB"
level=info ts=2026-02-16T14:39:20.643738144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:39:20.647574472Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:39:21.768968473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:39:22.980808871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:39:24.017864717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:39:24.02126436Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:39:24.270164307Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-02-16T14:39:24.746370531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:39:24.749577012Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:39:26.475379368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:39:26.478852949Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:39:26.768923001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:39:27.391233716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:39:28.019269062Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:39:29.853022364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:39:29.857924907Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:39:29.857947207Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:29.857973496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:39:29.862553461Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:39:31.769057203Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:39:31.769091643Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:39:31.769105225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:39:31.769112174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:39:31.769259445Z caller=checkpoint.go:378 msg="writing series" size="377 kB"
level=debug ts=2026-02-16T14:39:31.769273651Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-02-16T14:39:32.720123002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:39:35.52010803Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=warn ts=2026-02-16T14:39:36.768813563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:37.424604465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="57 kB" avg_comp="19 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:39:37.428720006Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:39:37.938624267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:39:37.942094647Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:39:38.869172592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:39:38.872564571Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:39:39.270088957Z caller=checkpoint.go:378 msg="writing series" size="475 kB"
level=info ts=2026-02-16T14:39:40.509460131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:39:40.512971042Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:39:40.512994005Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:40.513016896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:39:40.516247796Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:39:41.014762215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:39:41.018128427Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:39:41.018149432Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:41.018170436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:39:41.020924466Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:39:41.595779952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:39:41.599597501Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:39:41.630667508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:39:41.634380658Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:39:41.768866273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:42.319037126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:39:42.322340545Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:39:43.02326441Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T14:39:43.023289633Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.6 MB" series=3
level=info ts=2026-02-16T14:39:44.366787315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:39:44.370344814Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:39:44.370367078Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:44.370392558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:39:44.373252305Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:39:45.914343556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:39:45.917725059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:39:46.76947975Z caller=checkpoint.go:378 msg="writing series" size="897 B"
level=warn ts=2026-02-16T14:39:46.769479826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:48.284386034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:39:48.290437667Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:39:49.038045377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:39:49.04260892Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:39:49.747311963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:39:49.753041087Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T14:39:50.519979384Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T14:39:50.625396172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:39:50.628937655Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:39:51.077547139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:39:51.081266399Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:39:51.081292786Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:51.081317983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:39:51.084318607Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:39:51.522555697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:39:51.526364209Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:39:51.526385941Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:39:51.52642382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:39:51.529181708Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:39:51.7686899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:39:52.402635848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:39:52.406237271Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:39:52.749764432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:39:52.753268744Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:39:53.882228225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:39:53.885742848Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:39:54.269873603Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T14:39:56.769255427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:39:57.393352925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:39:58.019727569Z caller=checkpoint.go:378 msg="writing series" size="762 kB"
level=info ts=2026-02-16T14:39:58.706115472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:39:58.709744268Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:39:59.875356378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:39:59.878995824Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:40:00.053545057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:40:00.057469144Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:40:00.447589835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:40:00.454419402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:40:00.642013507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:40:00.645978295Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:40:01.748054923Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:40:01.748056707Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:40:01.768662854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:40:01.76869393Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:40:01.768702279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:40:01.768698577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:01.769836107Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:40:03.161319537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:40:03.16459947Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:40:03.348224231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:40:03.351856935Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:40:04.509607916Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:40:05.164763523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:40:05.168007584Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:40:05.51994392Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-02-16T14:40:06.76869464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:40:07.289102688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:40:07.294926409Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:40:08.560002795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="157 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:40:08.564279314Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:40:09.269263408Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T14:40:11.247067414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:40:11.250556792Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:40:11.768980041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:13.019235948Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=debug ts=2026-02-16T14:40:13.019261122Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-02-16T14:40:13.855834355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:40:13.860948122Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:40:16.382490664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:40:16.385732965Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:40:16.768759298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:16.769818834Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-02-16T14:40:17.114508767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:40:20.322101266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:40:20.325647673Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:40:20.523022273Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T14:40:20.52304585Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=2
ts=2026-02-16T14:40:21.252261687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60562"
level=warn ts=2026-02-16T14:40:21.769502568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:24.26972738Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:40:24.504148714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:40:24.507693182Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:40:25.010224648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:40:25.013581517Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T14:40:25.943127197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:46414"
level=warn ts=2026-02-16T14:40:26.768725731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:40:27.303138518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:40:27.306928604Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T14:40:27.395998496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:40:27.523233131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:40:27.527328848Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:40:27.873594525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:40:27.877327074Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:40:28.020364394Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=info ts=2026-02-16T14:40:28.766304874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:40:28.769875203Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:40:29.653076789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:40:29.657214428Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:40:30.748476478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:40:31.356250892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:40:31.359645684Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:40:31.383715928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:40:31.386994366Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:40:31.769217641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:40:31.769234487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:31.769254107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:40:31.769266994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:40:31.769293083Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:40:35.21159214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=3 total_comp="57 kB" avg_comp="19 kB" total_uncomp="541 kB" avg_uncomp="180 kB" full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:40:35.215127066Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:40:35.440801056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:40:35.444643538Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:40:35.519825313Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-02-16T14:40:35.519847426Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-02-16T14:40:36.500880354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:40:36.504448156Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:40:36.769147851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:40:37.15662958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:40:37.159818433Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:40:38.823937911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:40:38.827350901Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:40:39.269875592Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-02-16T14:40:41.769370933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:43.019729566Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=info ts=2026-02-16T14:40:43.084834699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:40:43.088325236Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:40:43.260856366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:40:43.264348458Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:40:46.769204231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:40:46.769291422Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=debug ts=2026-02-16T14:40:50.519935746Z caller=checkpoint.go:378 msg="writing series" size="440 kB"
level=debug ts=2026-02-16T14:40:50.519960361Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-02-16T14:40:50.576153753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:40:50.583014461Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:40:51.768604616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:40:52.341575557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:40:52.34849175Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:40:53.631804329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:40:53.637384166Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:40:53.637431844Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:40:53.637457249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:40:53.642715179Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:40:54.270393803Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:40:54.270440367Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-02-16T14:40:54.447465096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:40:54.451615992Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:40:55.313175889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:40:55.317576406Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:40:55.857819833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="157 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:40:55.862339431Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T14:40:55.944357597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:47738"
level=info ts=2026-02-16T14:40:56.119928785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:40:56.124899529Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T14:40:56.768791363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:40:57.397194692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:40:58.019734302Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T14:40:58.30270538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:40:58.306169496Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:40:59.957626441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:40:59.961486255Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:41:01.747803811Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:41:01.748859175Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-02-16T14:41:01.769329667Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:41:01.769366565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:41:01.76939254Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:41:01.769417454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:41:01.769471558Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=debug ts=2026-02-16T14:41:03.405464524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:41:04.685540278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:41:04.692138981Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:41:05.427483793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:41:05.431238433Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:41:05.519502491Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=warn ts=2026-02-16T14:41:06.768698798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:41:09.269880676Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T14:41:09.667386878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:41:09.670862972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:41:10.971389057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:41:11.768803967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:41:13.019650641Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-02-16T14:41:13.904831891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:41:13.908734342Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:41:14.451463002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:41:14.455234982Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:41:14.716643445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:41:14.720196399Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:41:15.468561416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:41:15.472148017Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:41:16.768595658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:41:16.769712862Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T14:41:18.634032813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:41:18.637477659Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:41:19.869634523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:41:19.87307649Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:41:20.51988735Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:41:20.929200485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:41:20.93377972Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:41:21.768684395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:41:23.499096315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:41:23.502544909Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:41:23.895841495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:41:23.899243133Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:41:24.269373981Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:41:24.437709144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:41:24.441291604Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:41:25.327838452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:41:25.33161536Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:41:25.459354073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" idle=1 full=3 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:41:25.462743023Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:41:25.597114777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:41:25.600621347Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:41:26.769460493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:41:27.00631241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:41:27.010482721Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T14:41:27.399289249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:41:28.019481188Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-02-16T14:41:31.769140019Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:41:31.769143275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:41:31.769152323Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=debug ts=2026-02-16T14:41:31.769167925Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:41:31.769177133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:41:32.718000681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:41:32.72609678Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:41:32.741295923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:41:32.744806616Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:41:32.744829265Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:41:32.744852718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:41:32.748082148Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:41:34.759019598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:41:34.762530981Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:41:35.519569516Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:41:36.082455015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:41:36.086074525Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:41:36.769301378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:41:37.942534933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="157 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:41:37.947030457Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:41:39.170579903Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:41:39.26993481Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=info ts=2026-02-16T14:41:40.529369186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:41:40.533074817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:41:41.768616852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:41:42.811176726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:41:42.818434034Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:41:43.020145209Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=debug ts=2026-02-16T14:41:43.020170087Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-02-16T14:41:44.376329485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:41:44.379992047Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:41:45.937711477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:41:45.941577047Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=warn ts=2026-02-16T14:41:46.768776895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:41:46.769868324Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:41:47.851472325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:41:47.856936698Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T14:41:50.519661897Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T14:41:51.769457555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:41:53.088029771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:41:53.094102756Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:41:54.26945845Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-02-16T14:41:55.947444476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:44548"
level=warn ts=2026-02-16T14:41:56.769255265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:41:56.950249271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:41:57.401193618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:41:58.020380476Z caller=checkpoint.go:378 msg="writing series" size="622 kB"
level=info ts=2026-02-16T14:41:58.680982337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:41:58.684679705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:41:59.290335038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:41:59.294100623Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:42:01.748052879Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:42:01.748236058Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:42:01.768794845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:42:01.768819844Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:42:01.76882753Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:42:01.768844676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:42:01.769983049Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=info ts=2026-02-16T14:42:04.179379019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:42:04.183487386Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:42:04.903798004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:42:04.907418232Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:42:05.415031213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" idle=1 full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:42:05.418641349Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:42:05.519980962Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=info ts=2026-02-16T14:42:05.591345075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:42:05.594694093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-02-16T14:42:06.769000435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:42:07.211849486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:42:07.215091497Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:42:08.28803708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:42:08.291274321Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:42:08.757534695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:42:08.757558678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:42:08.763081853Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T14:42:08.892318772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:42:08.895695107Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:42:08.895714426Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:42:09.058264682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:42:09.062271422Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:42:09.269993876Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:42:09.660689107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:42:09.664641546Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:42:09.751088376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:42:09.754732623Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:42:10.586200537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:42:10.589508247Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:42:11.768732944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:42:13.02009799Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=debug ts=2026-02-16T14:42:13.020121384Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-02-16T14:42:13.276742507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:42:13.280110252Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:42:15.255683944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:42:15.259250435Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:42:16.460558734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="128 kB" avg_uncomp="128 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:42:16.463992923Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:42:16.769201071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:42:16.769235128Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:42:17.269623907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:42:17.273249229Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:42:18.829463209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="441 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:42:18.83293712Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:42:19.3567575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:42:19.360069846Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:42:19.936311124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:42:19.941598907Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T14:42:20.389737397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="48 kB" avg_comp="12 kB" total_uncomp="311 kB" avg_uncomp="78 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:42:20.393365271Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:42:20.519687608Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:42:20.898925396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:42:20.902201375Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:42:21.769385819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:42:22.471835159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:42:22.475290282Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T14:42:23.930585419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57302"
level=debug ts=2026-02-16T14:42:24.269218036Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=info ts=2026-02-16T14:42:25.774450091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:42:25.779277803Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T14:42:25.949135221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:36062"
level=warn ts=2026-02-16T14:42:26.768605512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:42:27.403469568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:42:28.02003188Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-02-16T14:42:28.128326626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:42:28.135947279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:42:31.768684363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:42:31.768702909Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:42:31.768725556Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:42:31.768733531Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:42:31.769886139Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-02-16T14:42:32.348772018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:42:32.3535359Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:42:32.421618628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="157 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:42:32.42566388Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:42:34.188824858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:42:34.192881866Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=debug ts=2026-02-16T14:42:35.519496159Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-02-16T14:42:36.768927973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:42:39.089874648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:42:39.09633903Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:42:39.269921595Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-02-16T14:42:41.768961625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:42:42.97463527Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:42:43.019742616Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:42:43.205379801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:42:43.209444003Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=debug ts=2026-02-16T14:42:45.889258434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:42:45.89097637Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T14:42:46.769456532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:42:46.769499673Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-02-16T14:42:48.501051219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:42:48.504634044Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:42:50.457894107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:42:50.461501507Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:42:50.520344897Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:42:50.520378666Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=10
level=info ts=2026-02-16T14:42:51.289764032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:42:51.294717564Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T14:42:51.427164143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:42:51.430396717Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:42:51.430440012Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:42:51.430464971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:42:51.433103178Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:42:51.746145057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:42:51.749382105Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:42:51.769302755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:42:54.302795013Z caller=checkpoint.go:378 msg="writing series" size="32 MB"
level=debug ts=2026-02-16T14:42:54.302822401Z caller=checkpoint.go:390 msg="flushing series" totalSize="32 MB" series=1
ts=2026-02-16T14:42:55.950884537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:59926"
level=info ts=2026-02-16T14:42:56.417547132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" idle=1 full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:42:56.423075488Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:42:56.633726607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:42:56.637421137Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=warn ts=2026-02-16T14:42:56.76854025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:42:57.406074991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:42:58.019887788Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-02-16T14:42:58.044093007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:42:58.047652461Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:42:58.2381651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:42:58.242531245Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:42:58.299479843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:42:58.303017441Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:43:01.747882934Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:43:01.748958357Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:43:01.76844153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:43:01.76845405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:43:01.768469413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:43:01.768477924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:43:01.76974976Z caller=checkpoint.go:378 msg="writing series" size="505 kB"
level=info ts=2026-02-16T14:43:02.637679716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" idle=1 full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:43:02.643553189Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:43:02.860267151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:43:02.863734723Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:43:02.998086439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:43:03.001736173Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:43:03.858050245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:43:03.862047409Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:43:04.129126667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:43:04.132858328Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:43:04.793367621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:43:04.797538733Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:43:05.189667732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:43:05.193239391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:43:05.52003244Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:43:06.609397471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="14 kB" avg_comp="6.8 kB" total_uncomp="157 kB" avg_uncomp="78 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:43:06.612970841Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T14:43:06.768766523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:08.669654584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:43:08.673189888Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:43:08.673213705Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:43:08.67324801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:43:08.676373055Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:43:09.090652628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="446 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:43:09.094029587Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:43:09.269650001Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=info ts=2026-02-16T14:43:10.651054935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:43:10.654274278Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:43:11.768670839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:43:13.019670869Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T14:43:14.008533321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:43:14.01199009Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:43:14.647882656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:43:14.651270925Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:43:15.021212695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:43:16.768608428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:43:16.769715173Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-02-16T14:43:20.519372667Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-02-16T14:43:20.944698392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:43:21.76853866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:23.37718452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:43:23.3817252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T14:43:24.271071334Z caller=checkpoint.go:378 msg="writing series" size="2.7 MB"
level=debug ts=2026-02-16T14:43:24.271101403Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.5 MB" series=8
level=info ts=2026-02-16T14:43:24.849637386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:43:24.852915413Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T14:43:25.952677427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:44298"
level=warn ts=2026-02-16T14:43:26.769233195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:26.866764845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="470 kB" avg_comp="157 kB" total_uncomp="4.8 MB" avg_uncomp="1.6 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:43:26.870963756Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T14:43:27.408805315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:43:27.574306255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:43:27.582620586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:43:28.01918343Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T14:43:28.299916015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:43:28.303674466Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:43:28.303696526Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:43:28.30371857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:43:28.306806281Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:43:31.769097784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:43:31.769130537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T14:43:31.769124631Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.9 kB" series=1
level=info ts=2026-02-16T14:43:31.76914137Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:43:31.769153868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:31.776077343Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-02-16T14:43:31.777330677Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008272792s
level=info ts=2026-02-16T14:43:32.267709734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:43:32.273977358Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:43:32.622685347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:43:32.626142032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:43:36.768511004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:38.458680781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:43:38.462999228Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:43:41.223014159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:43:41.22727863Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:43:41.317533977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:43:41.321075989Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:43:41.600008353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:43:41.603908149Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:43:41.769235527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:43.286674216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:43:43.290175362Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:43:43.890812312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:43:43.894242471Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:43:44.547202819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:43:44.550697238Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:43:44.693614483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:43:44.696992623Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:43:44.764709949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:43:44.768152142Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:43:45.452915446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:43:45.457022548Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:43:45.457045657Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:43:45.457068034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:43:45.460056765Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:43:46.768627628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:47.337678633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:43:47.341199376Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:43:47.659361628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:43:47.663001625Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:43:47.715064115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:43:47.718488679Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:43:47.751961116Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:43:48.340269056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:43:48.343673307Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:43:48.683932508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:43:48.687336774Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:43:49.747430902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:43:49.752767329Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T14:43:49.801829673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:43:49.805650802Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T14:43:51.267076011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56394"
level=warn ts=2026-02-16T14:43:51.769387865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:43:56.313836167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:43:56.317883289Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:43:56.382240799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:43:56.386258261Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:43:56.768948987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:43:57.410266095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:43:58.43544851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:43:58.439898105Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:43:59.0398367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="14 kB" avg_comp="6.8 kB" total_uncomp="157 kB" avg_uncomp="78 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:43:59.043528578Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T14:43:59.323183244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="446 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:43:59.326977432Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:44:01.748106167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:44:01.748106933Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:44:01.768155213Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:44:01.768463515Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-02-16T14:44:01.768472921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:44:01.768493325Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:44:01.768503873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:44:01.768527529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:44:01.768723302Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=info ts=2026-02-16T14:44:04.323727891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:44:04.327069096Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:44:05.519314283Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-02-16T14:44:06.769104519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:44:08.48204848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:44:08.487036958Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:44:09.175865672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:44:09.269134663Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-02-16T14:44:11.768879866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:44:13.019662841Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-02-16T14:44:13.019686343Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=4
level=debug ts=2026-02-16T14:44:15.834308273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:44:16.768456095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:44:16.769574094Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T14:44:17.350312295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:44:17.355055095Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:44:17.369127965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" idle=1 full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:44:17.373359305Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:44:17.373387798Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:44:17.373435937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:44:17.377227315Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:44:17.631734602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:44:17.635510108Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:44:17.670598047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:44:17.678190876Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:44:17.678215046Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:44:17.678251669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:44:17.685448759Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:44:20.519463889Z caller=checkpoint.go:378 msg="writing series" size="568 kB"
level=warn ts=2026-02-16T14:44:21.768765888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:44:22.857812759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:44:22.861247057Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:44:23.399023108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:44:23.40215215Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:44:24.236509396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:44:24.240369756Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:44:24.269537807Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-02-16T14:44:26.63560957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:44:26.642460315Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:44:26.769388637Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:44:27.037996524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:44:27.041930106Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:44:27.048940506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:44:27.052313225Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T14:44:27.411786372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:44:28.019552874Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:44:28.042826286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:44:28.046371762Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:44:28.582626568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:44:28.586642839Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:44:29.52626127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:44:29.53004869Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:44:29.671303663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:44:29.674813517Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:44:30.980819133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:44:30.984148961Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:44:31.231933882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:44:31.23562281Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:44:31.508231137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:44:31.516305638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:44:31.769074532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:44:31.769094117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:44:31.769116848Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:44:31.769124727Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:44:31.76929692Z caller=checkpoint.go:378 msg="writing series" size="497 kB"
level=debug ts=2026-02-16T14:44:31.769310534Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-02-16T14:44:32.083456708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:44:32.086808155Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:44:33.965240271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:44:33.969531397Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:44:35.51912821Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=warn ts=2026-02-16T14:44:36.768752301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:44:39.26993774Z caller=checkpoint.go:378 msg="writing series" size="475 kB"
level=info ts=2026-02-16T14:44:39.310849355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:44:39.315490219Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:44:39.315524035Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:44:39.315551097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:44:39.319104044Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:44:39.70445675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:44:39.707839804Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:44:40.974171146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:44:40.978136127Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:44:41.768602437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:44:42.150562929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:44:42.308327722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:44:42.3123981Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:44:43.02328264Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-02-16T14:44:43.023307248Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.6 MB" series=3
level=info ts=2026-02-16T14:44:44.332898631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:44:44.336493345Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:44:44.484134834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:44:44.487668216Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:44:45.597278517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:44:45.600725571Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T14:44:46.768877589Z caller=checkpoint.go:378 msg="writing series" size="899 B"
level=warn ts=2026-02-16T14:44:46.768885581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:44:47.66052858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.8 kB" avg_comp="446 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:44:47.664060044Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:44:48.851962962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:44:48.857129837Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-02-16T14:44:50.519694521Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=warn ts=2026-02-16T14:44:51.769365877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:44:53.939174576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49448"
level=debug ts=2026-02-16T14:44:54.269488068Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T14:44:54.409916874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:44:54.417637427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:44:55.365155335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="14 kB" avg_comp="6.8 kB" total_uncomp="157 kB" avg_uncomp="78 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:44:55.368617611Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c66bacf63%3A19c66d096f6%3A6d6ac07c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T14:44:55.368638682Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:44:55.368662302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:44:55.371878412Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:44:55.958060267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:60528"
level=warn ts=2026-02-16T14:44:56.76903134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:44:57.413695422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:44:58.02009205Z caller=checkpoint.go:378 msg="writing series" size="955 kB"
level=info ts=2026-02-16T14:44:58.650469036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:44:58.65514399Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:44:59.649221251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:44:59.652957694Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:45:01.485908228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:45:01.489919376Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:45:01.747454614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:45:01.748518026Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:45:01.748713132Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T14:45:01.748754131Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968058 n=1
level=debug ts=2026-02-16T14:45:01.748795964Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968058 n=1
level=info ts=2026-02-16T14:45:01.768766978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:45:01.768781337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:45:01.768798249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T14:45:01.768796586Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:45:01.768818841Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:45:01.777198385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:45:02.721956619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:45:02.725396231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:45:05.518716828Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-02-16T14:45:05.983339709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:45:05.987304144Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:45:06.768489183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:45:07.024962338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:45:07.029758878Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:45:09.253534989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:45:09.257904328Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:45:09.269089984Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=debug ts=2026-02-16T14:45:09.26911407Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-02-16T14:45:10.099198971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:45:10.104388066Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:45:11.698967993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:45:11.75696985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="767 B" avg_comp="767 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:45:11.760870312Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:45:11.769008707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:45:12.604674992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:45:12.610497775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:45:12.620529423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="522 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:45:12.623705385Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:45:12.623725348Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:12.623748196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:45:12.6266436Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:45:13.018986002Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=info ts=2026-02-16T14:45:14.264372731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="715 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:45:14.26791531Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:45:14.267940484Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:14.267964955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:45:14.271082485Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:45:16.27074647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:45:16.274385079Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:45:16.29945092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:45:16.302501221Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:45:16.769043848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:45:16.769113776Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:45:17.850591285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:45:17.853890104Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T14:45:17.853910528Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:17.85393255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:45:17.856822699Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:45:19.792545386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:45:19.795936117Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:45:19.795957086Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:19.795977174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:45:19.798627053Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:45:20.528571657Z caller=checkpoint.go:378 msg="writing series" size="8.3 MB"
level=debug ts=2026-02-16T14:45:20.528597346Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.6 MB" series=3
level=info ts=2026-02-16T14:45:21.313137595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:45:21.316751851Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=error ts=2026-02-16T14:45:21.316777827Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:21.316811238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:45:21.319576113Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:45:21.768902812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:45:23.987199498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:45:23.990728555Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:45:23.990750035Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:23.990773805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:45:23.993670882Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:45:24.269186937Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:45:24.73861941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="512 B" avg_comp="256 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:45:24.742237718Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:45:25.979952707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:45:25.98332541Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:45:26.279141628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:45:26.283266148Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T14:45:26.768688062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:45:27.415211261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:45:28.01932762Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T14:45:28.019353607Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-02-16T14:45:28.810377616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="382 B" total_uncomp="1.7 kB" avg_uncomp="343 B" idle=1 full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:45:28.814782061Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:45:28.814805177Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:28.814828745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="382 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:45:28.819105541Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:45:28.890330667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:45:28.893601352Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:45:28.893619437Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:29.115248065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:45:29.119217002Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:45:29.139324607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:45:29.142589981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:45:29.143094613Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:45:29.145748748Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:45:29.243060199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:45:29.247497251Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:45:29.676307579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:45:29.679878336Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-02-16T14:45:31.768928936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:45:31.768960607Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T14:45:31.768950619Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T14:45:31.768969781Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:45:31.768986913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:45:32.870059393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:45:32.873646927Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:45:35.519634855Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T14:45:35.772708803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:45:35.776343623Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=warn ts=2026-02-16T14:45:36.768969961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:45:36.772233007Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:45:39.269277131Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-02-16T14:45:39.565539047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:45:39.570016273Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:45:39.635173825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:45:39.639011598Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:45:39.723067012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:45:39.726679576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:45:39.726701512Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:39.726725845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:45:39.730133001Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:45:39.847125806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:45:39.850972285Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:45:41.566535204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:45:41.573146045Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T14:45:41.768703386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:45:42.62294109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:45:42.626385142Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=debug ts=2026-02-16T14:45:43.018896348Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=info ts=2026-02-16T14:45:43.401907599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:45:43.405360316Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:45:43.891703808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:45:43.899053481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:45:44.712526326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="382 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:45:44.716250145Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:45:45.225189278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:45:45.228782819Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:45:46.251036474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:45:46.254435473Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:45:46.254457651Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:46.254480549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:45:46.257165847Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T14:45:46.7686617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:45:46.768726134Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=debug ts=2026-02-16T14:45:50.519316818Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=debug ts=2026-02-16T14:45:50.519340734Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-02-16T14:45:51.768855742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:45:54.270307011Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T14:45:54.270332323Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-02-16T14:45:54.91272914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:45:54.917229196Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:45:55.852098701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:45:55.855665451Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:45:55.896749534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:45:55.900698859Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T14:45:55.96071977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:34760"
level=warn ts=2026-02-16T14:45:56.769092619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:45:56.904422003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:45:56.907950079Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
ts=2026-02-16T14:45:57.41672548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=debug ts=2026-02-16T14:45:57.620302119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:45:58.019720322Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T14:45:58.619328657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:45:58.622717871Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:45:58.637915457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:45:58.64106469Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:45:58.641084196Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:58.641105592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:45:58.643932103Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:45:59.798980546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:45:59.804197902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:45:59.804222274Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:45:59.804247543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:45:59.808115672Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:46:01.748044779Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:46:01.748253041Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:46:01.768427603Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:46:01.76846589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:46:01.7684746Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:46:01.768480383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:46:01.769610924Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=info ts=2026-02-16T14:46:02.402091842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:46:02.405651703Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:46:03.099719999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:46:03.103450025Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:46:03.994186665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:46:03.997630119Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:46:04.844352449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:46:04.846204435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T14:46:04.847670679Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=info ts=2026-02-16T14:46:05.220631217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:46:05.224234057Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:46:05.5189744Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=info ts=2026-02-16T14:46:06.357370975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:46:06.36068356Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:46:06.76931502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:46:08.344570567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:46:08.348464676Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:46:09.269280572Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T14:46:10.973398706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:46:10.979992389Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:46:11.588960102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:46:11.594602116Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T14:46:11.769099324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:46:12.546804884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="530 B" avg_comp="265 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:46:12.550463312Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66bb381e%3A19c66bb565a%3Abdbfb4d8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:46:12.550484472Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:46:12.550508497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="94 kB" avg_comp="94 kB" total_uncomp="936 kB" avg_uncomp="936 kB" idle=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-02-16T14:46:12.553586724Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19c66bb381e%3A19c66de6c2e%3A2a0d53f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-02-16T14:46:13.019116329Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-02-16T14:46:14.735837161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:46:14.740484905Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:46:15.469049613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:46:15.472961254Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=warn ts=2026-02-16T14:46:16.768602986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:46:16.768685732Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T14:46:17.227811528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:46:17.231503512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:46:17.982606715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:46:17.986178564Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:46:17.986199318Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:46:17.986219719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:46:17.989139776Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:46:18.021253483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:46:18.027834143Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:46:20.494588281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=2 total_comp="4.4 kB" avg_comp="2.2 kB" total_uncomp="23 kB" avg_uncomp="11 kB" full=2 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:46:20.498187224Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c66bb381e%3A19c66bb565a%3Afdbf7e51?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T14:46:20.498207881Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:46:20.498231337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:46:20.501192883Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:46:20.51929989Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:46:21.732442046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="382 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:46:21.735896742Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:46:21.768763324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:46:22.174358381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:46:22.177511154Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:46:24.268722835Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T14:46:25.245828901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:46:25.249495227Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:46:26.548188873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:46:26.551709133Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T14:46:26.769159317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:46:27.418959796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:46:27.771128308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:46:27.774593007Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:46:28.018785673Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-02-16T14:46:28.139242739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:46:28.142603671Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:46:28.142623446Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:46:28.14264633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:46:28.145526803Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:46:28.692878858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="105 kB" avg_comp="105 kB" total_uncomp="936 kB" avg_uncomp="936 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-02-16T14:46:28.69646519Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19c66bb381e%3A19c66de6c2e%3A2a0d53f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-02-16T14:46:29.232257003Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:46:29.932965101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:46:29.940015637Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-02-16T14:46:30.522841775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:46:30.526465846Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:46:31.768851164Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:46:31.768851166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:46:31.768880514Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T14:46:31.768870366Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-02-16T14:46:31.768890879Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:46:35.519439278Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T14:46:36.761875043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:46:36.765119625Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:46:36.769185779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:46:39.269396269Z caller=checkpoint.go:378 msg="writing series" size="295 kB"
level=info ts=2026-02-16T14:46:39.291436822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:46:39.295457799Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T14:46:41.586619328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:46:41.594894209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:46:41.769200776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:46:41.788382024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:46:41.796517342Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:46:42.238432676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:46:42.240185935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:46:42.354675254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:46:42.358225035Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:46:43.019297202Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=info ts=2026-02-16T14:46:46.386207264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:46:46.389563794Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:46:46.7688256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:46:46.768836482Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:46:47.615802102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:46:47.62003423Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:46:47.820519731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:46:47.823962874Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:46:49.492196482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="105 kB" avg_comp="105 kB" total_uncomp="936 kB" avg_uncomp="936 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-02-16T14:46:49.495914174Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19c66bb381e%3A19c66de6c2e%3A2a0d53f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-02-16T14:46:49.820907992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:46:49.824718268Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:46:50.519299073Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T14:46:50.92258743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:46:50.926240656Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:46:51.768546044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:46:52.774347095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:46:52.777998417Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:46:54.269494846Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-02-16T14:46:54.269519851Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=info ts=2026-02-16T14:46:55.271813787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:46:55.275917156Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T14:46:55.963687539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:60918"
level=info ts=2026-02-16T14:46:56.222523722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:46:56.225988147Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:46:56.278184296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:46:56.282664667Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T14:46:56.768536136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:46:57.160992596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:46:57.16464849Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T14:46:57.420279003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=debug ts=2026-02-16T14:46:58.019741675Z caller=checkpoint.go:378 msg="writing series" size="626 kB"
level=info ts=2026-02-16T14:46:58.731050699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:46:58.735010631Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:46:59.717543567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:46:59.721141264Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=debug ts=2026-02-16T14:47:01.736729233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:47:01.747552631Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:47:01.748605165Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:47:01.769064712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:47:01.7690899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T14:47:01.76908235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:01.769098425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:47:01.769217169Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=info ts=2026-02-16T14:47:02.788517904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:47:02.792867008Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:47:03.372453361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.5 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:47:03.376247927Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T14:47:05.519742988Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=warn ts=2026-02-16T14:47:06.768661519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:06.7788122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:47:06.78509674Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:47:07.218831329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:47:07.222493428Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=info ts=2026-02-16T14:47:08.153100422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:47:08.157399387Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T14:47:09.269607297Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:47:11.321942465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:47:11.325644944Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=warn ts=2026-02-16T14:47:11.769359021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:12.246011436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:47:12.251314105Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:47:13.019820475Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=debug ts=2026-02-16T14:47:13.019845859Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-02-16T14:47:15.887105035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:47:15.893803822Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T14:47:16.768926195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:47:16.768978238Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-02-16T14:47:19.634523675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:47:19.63848912Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:47:19.959450841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:47:19.963023367Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:47:20.519041941Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:47:21.610293163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="378 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:47:21.613548874Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:47:21.769089643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:21.772230739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:47:21.775387936Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:47:23.757166306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:47:23.764586581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:47:24.268853858Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-02-16T14:47:25.342251012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T14:47:26.335519335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:47:26.339426079Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T14:47:26.769002998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:47:27.421923523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f69d88a4 10.129.0.60:7946"
level=debug ts=2026-02-16T14:47:28.019624165Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T14:47:30.626391113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:47:30.629923952Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T14:47:31.7691776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:47:31.769185986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:47:31.769206901Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:47:31.769215987Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:47:31.769339659Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=info ts=2026-02-16T14:47:35.462870147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:47:35.467074597Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:47:35.513227264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:47:35.516657236Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:47:35.518774975Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=debug ts=2026-02-16T14:47:35.938457291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:47:36.769362791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:38.119529051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:47:38.123236573Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:47:38.230746106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:47:38.234254676Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:47:38.378703248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:47:38.382352625Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:47:39.268978945Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-02-16T14:47:39.697806014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:47:39.701902301Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:47:40.997356432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:47:41.002059417Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T14:47:41.769185183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:47:43.019607651Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T14:47:43.197350756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="105 kB" avg_comp="105 kB" total_uncomp="936 kB" avg_uncomp="936 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-02-16T14:47:43.20137832Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19c66bb381e%3A19c66de6c2e%3A2a0d53f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-02-16T14:47:43.797099594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:47:43.801151798Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:47:43.87137637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:47:43.874670099Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:47:44.647051613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:47:44.650527158Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:47:44.650553026Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:47:44.650577381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:47:44.653317283Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:47:46.700993963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:47:46.704634876Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T14:47:46.768916145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:47:46.768967232Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-02-16T14:47:50.521195401Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T14:47:50.521218795Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=10
level=warn ts=2026-02-16T14:47:51.769209207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:52.217054899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="456 kB" avg_comp="456 kB" total_uncomp="6.3 MB" avg_uncomp="6.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T14:47:52.22168023Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c66bb381e%3A19c66c36b54%3Ac5a42fc7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T14:47:53.437958376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:47:53.441555657Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:47:53.567975656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:47:53.571145661Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:47:54.289666076Z caller=checkpoint.go:378 msg="writing series" size="32 MB"
level=debug ts=2026-02-16T14:47:54.289693229Z caller=checkpoint.go:390 msg="flushing series" totalSize="32 MB" series=1
level=info ts=2026-02-16T14:47:55.630759176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:47:55.634529281Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
ts=2026-02-16T14:47:55.967545795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.58:51984"
level=warn ts=2026-02-16T14:47:56.768951831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:47:56.859240438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=3 total_comp="3.5 kB" avg_comp="1.2 kB" total_uncomp="11 kB" avg_uncomp="3.7 kB" full=3 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:47:56.8628751Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66baf7cf%3A19c66baf7d0%3A9748e38c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"application-service\"}"
ts=2026-02-16T14:47:57.424122226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-6wnwh-842829c1 10.130.0.54:7946"
level=info ts=2026-02-16T14:47:57.635856131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:47:57.63991769Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:47:58.019578568Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-02-16T14:47:59.698993455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.6 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:47:59.702597675Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:48:01.748040506Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:48:01.748244547Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.71:11211 count=1
level=info ts=2026-02-16T14:48:01.768978343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:48:01.769019936Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:48:01.769032808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:48:01.769033209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:48:01.769301279Z caller=checkpoint.go:378 msg="writing series" size="512 kB"
level=info ts=2026-02-16T14:48:02.028456777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:48:02.031984958Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:48:04.493054749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:48:04.496715949Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66bb5659%3A19c66bbcabf%3A429c5f5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:48:05.441316772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:48:05.447026027Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:48:05.447051393Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:48:05.447079397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:48:05.452483916Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:48:05.519564872Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:48:06.657939488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:48:06.662660812Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"
level=warn ts=2026-02-16T14:48:06.768671269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:48:08.790298867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T14:48:09.269733663Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=info ts=2026-02-16T14:48:10.257676606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-02-16T14:48:10.263574608Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19c66bb3800%3A19c66e6151d%3Ac7c26f70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-02-16T14:48:11.768819008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:48:12.85953835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="97 kB" avg_uncomp="49 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T14:48:12.863276214Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c66baf7cf%3A19c66c299f4%3A7cbf5f3a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T14:48:13.019747007Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T14:48:13.622369147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="8.9 kB" avg_comp="8.9 kB" total_uncomp="44 kB" avg_uncomp="44 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:48:13.627826935Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66bacf63%3A19c66baf7d0%3Acb2ea3bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:48:16.486878823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="531 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:48:16.490658254Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66bb4af1%3A19c66bb4af2%3Af5431396?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:48:16.599009945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:48:16.602531575Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66baf7cf%3A19c66bb4af2%3A3432b94c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:48:16.768817177Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-02-16T14:48:16.768898591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:48:18.970676055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.2 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:48:18.977764154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66bacb05%3A19c66be1f88%3A1d074f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:48:19.63427953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=5 total_comp="1.9 kB" avg_comp="382 B" total_uncomp="1.7 kB" avg_uncomp="343 B" full=5 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:48:19.637600203Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0fbe3%3A19c66c0fbe4%3A3f3c498b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:48:19.708655908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:48:19.71416089Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:48:20.51872184Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
ts=2026-02-16T14:48:21.286260618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33624"
level=warn ts=2026-02-16T14:48:21.769329931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:48:22.504690896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=2 total_comp="483 kB" avg_comp="241 kB" total_uncomp="3.6 MB" avg_uncomp="1.8 MB" full=1 synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-02-16T14:48:22.508127156Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19c66bb4b11%3A19c66bb4b12%3Acd749f50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"integration-service\"}"
level=info ts=2026-02-16T14:48:22.608538753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=4 total_comp="57 kB" avg_comp="14 kB" total_uncomp="551 kB" avg_uncomp="138 kB" full=4 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T14:48:22.612170005Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c66bb381e%3A19c66bb381f%3A70d5b085?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T14:48:24.270880133Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-02-16T14:48:24.270904985Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=8
level=debug ts=2026-02-16T14:48:24.325380278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T14:48:26.769120801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:48:27.425812416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-hb8bn-ba843256 10.129.0.58:7946"
level=info ts=2026-02-16T14:48:27.99446284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="105 kB" avg_comp="105 kB" total_uncomp="936 kB" avg_uncomp="936 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-02-16T14:48:27.998239205Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19c66bb381e%3A19c66de6c2e%3A2a0d53f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-02-16T14:48:28.01434319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.7 kB" avg_comp="5.7 kB" total_uncomp="27 kB" avg_uncomp="27 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:48:28.018534336Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66baf7cf%3A19c66baf7d0%3Adb2b10e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:48:28.019459353Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T14:48:29.745159149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=4 total_comp="470 kB" avg_comp="118 kB" total_uncomp="4.8 MB" avg_uncomp="1.2 MB" full=4 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:48:29.749157881Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66baef48%3A19c66c61494%3A27ef612?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:48:31.768798133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:48:31.768816719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:48:31.768835042Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-02-16T14:48:31.768834529Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.9 kB" series=1
level=info ts=2026-02-16T14:48:31.768847126Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:48:31.775123707Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-02-16T14:48:31.776435424Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007828271s
level=info ts=2026-02-16T14:48:32.311047684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:48:32.314497705Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c32a7b%3A19c66c511bf%3Ab379f2da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:48:32.962804946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.4 kB" avg_comp="724 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:48:32.966206061Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66baf7cf%3A19c66bb565a%3Aa01a6b1a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:48:33.407372953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="785 B" avg_comp="785 B" total_uncomp="2.9 kB" avg_uncomp="2.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:48:33.410892752Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66bb8a7c%3A19c66bdd856%3A376fbf37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:48:34.935738574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="66 kB" avg_comp="66 kB" total_uncomp="424 kB" avg_uncomp="424 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:48:34.939118354Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66bb4af0%3A19c66bd4e7d%3Aab50f0ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:48:35.389362621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=007219f0945bf53d immediate=false num_chunks=1 total_comp="88 kB" avg_comp="88 kB" total_uncomp="384 kB" avg_uncomp="384 kB" full=1 labels="{stream=\"chains-e2e-kctk\"}"
level=error ts=2026-02-16T14:48:35.392837148Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7219f0945bf53d/19c66c2c60c%3A19c66cca565%3Abf5e9ad5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-kctk\"}"
level=info ts=2026-02-16T14:48:35.600538024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="894 kB" avg_comp="894 kB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:48:35.604996428Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66baf7cf%3A19c66d9553e%3A9d887604?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T14:48:36.768977355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:48:38.013695151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:48:38.791019998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:48:38.794597023Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66bb381e%3A19c66bb78b6%3A9e00e2f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:48:40.357372606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="48 kB" avg_comp="9.6 kB" total_uncomp="311 kB" avg_uncomp="62 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T14:48:40.361097153Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c66bacad3%3A19c66c5f836%3Ab3364c2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T14:48:40.934698003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:48:40.93827293Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66baf7cf%3A19c66bb4af2%3Add12c68f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:48:41.769329325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:48:44.324398273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:48:44.331388187Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66bac689%3A19c66cdbabe%3Ac17216b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:48:45.485475218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:48:45.489018437Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:48:45.489045842Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:48:45.489070533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:48:45.492011439Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66baf7cf%3A19c66bdff65%3Acebf07df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:48:46.335364378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=990a1a30d435c0a7 immediate=false num_chunks=1 total_comp="216 kB" avg_comp="216 kB" total_uncomp="833 kB" avg_uncomp="833 kB" full=1 labels="{stream=\"build-e2e-khfp\"}"
level=error ts=2026-02-16T14:48:46.339189644Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/990a1a30d435c0a7/19c66c3a675%3A19c66cc1f75%3Af77b5d7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-khfp\"}"
level=warn ts=2026-02-16T14:48:46.769095262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:48:47.133189217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=038966c83b7de7e5 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="665 kB" avg_uncomp="665 kB" full=1 labels="{stream=\"integration2-isns\"}"
level=error ts=2026-02-16T14:48:47.137176555Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/38966c83b7de7e5/19c66d4fe1c%3A19c66dcfcfd%3A6f57c3c4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-isns\"}"