level=info ts=2026-02-16T11:30:57.444143816Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T11:30:57.444175088Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T11:30:57.444591729Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T11:30:57.445699942Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T11:30:57.446091898Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:30:57.446162178Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.60 inf=eth0
level=info ts=2026-02-16T11:30:57.446671493Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-d875fc63
ts=2026-02-16T11:30:57.446738386Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T11:30:57.446751045Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.60 advertisePort=7946
level=debug ts=2026-02-16T11:30:57.446773841Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.60 advertisePort=7946
level=info ts=2026-02-16T11:30:57.446883969Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T11:30:57.451260979Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:30:57.451285265Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-02-16T11:30:57.452548671Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T11:30:57.452727832Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-02-16T11:30:57.452753305Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:30:57.452768476Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:30:57.452783672Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.901464ms
level=info ts=2026-02-16T11:30:57.453067769Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T11:30:57.453085414Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T11:30:57.453084052Z
level=debug ts=2026-02-16T11:30:57.453107743Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=error ts=2026-02-16T11:30:57.453108356Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T11:30:57.453123852Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T11:30:57.453138777Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T11:30:57.453144725Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T11:30:57.453143863Z
level=debug ts=2026-02-16T11:30:57.453151736Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T11:30:57.453163452Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T11:30:57.454493832Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:30:57.454562984Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.60 inf=eth0
ts=2026-02-16T11:30:57.455478698Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:30:57.455489455Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:30:57.467402625Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.467414875Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=info ts=2026-02-16T11:30:57.467426646Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T11:30:57.467423507Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-02-16T11:30:57.467430879Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T11:30:57.467448079Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.467447315Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T11:30:57.467459306Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T11:30:57.467473692Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:57.467477088Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T11:30:57.467670596Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-02-16T11:30:57.46767324Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-02-16T11:30:57.467677786Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-02-16T11:30:57.46768049Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T11:30:57.467681282Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T11:30:57.467685714Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T11:30:57.467684257Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-02-16T11:30:57.467691226Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.467695273Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-02-16T11:30:57.467696326Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-02-16T11:30:57.467718983Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T11:30:57.467746875Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T11:30:57.46782502Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:30:57.467829159Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=info ts=2026-02-16T11:30:57.467831924Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T11:30:57.467851074Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-02-16T11:30:57.467874858Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-02-16T11:30:57.467922613Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=71.783µs errors=false
level=info ts=2026-02-16T11:30:57.467931937Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-02-16T11:30:57.468073755Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=223.462µs errors=false
level=info ts=2026-02-16T11:30:57.46808328Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T11:30:57.468092695Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=242.659µs
level=info ts=2026-02-16T11:30:57.468128639Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T11:30:57.46813787Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T11:30:57.46814868Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-02-16T11:30:57.46816109Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=13.225615694s
level=debug ts=2026-02-16T11:30:57.46823487Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T11:30:57.468241063Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-02-16T11:30:57.468247263Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=info ts=2026-02-16T11:30:57.468258327Z caller=loki.go:599 msg="Loki started" startup_time=84.236805ms
level=debug ts=2026-02-16T11:30:57.468280575Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-02-16T11:30:58.545412773Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:30:58.545442593Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:00.977931928Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:00.977960035Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:31:02.468594518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:31:05.445777698Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:05.44582129Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:31:07.468518887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:31:12.468591746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:31:13.540867331Z caller=logging.go:137 msg="GET /ready (503) 150.885µs"
ts=2026-02-16T11:31:14.237261546Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.59:7946: dial tcp 10.130.0.59:7946: connect: connection refused"
ts=2026-02-16T11:31:14.238775623Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused"
ts=2026-02-16T11:31:14.240477314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.69:7946"
level=info ts=2026-02-16T11:31:14.241928526Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=16.788758475s
level=warn ts=2026-02-16T11:31:17.469298621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:31:22.468355081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:31:23.539107399Z caller=logging.go:137 msg="GET /ready (503) 103.273µs"
level=info ts=2026-02-16T11:31:27.468928771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T11:31:27.468956943Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T11:31:27.468965855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:31:27.469009918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:31:29.019737143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=warn ts=2026-02-16T11:31:32.469165915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:31:37.468497193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:31:42.468311808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:31:44.546385475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T11:31:45.365225811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:60330"
level=warn ts=2026-02-16T11:31:47.469126231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:31:52.469170869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:31:53.935915215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T11:31:57.452920055Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:31:57.453369929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:31:57.468566749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:31:57.468605724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:31:57.468615874Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:31:57.468605208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:31:59.022502996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T11:32:00.130860036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48896"
level=warn ts=2026-02-16T11:32:02.468714031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:32:07.46906744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:32:12.468785823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:32:12.485389895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:32:14.220778933Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T11:32:15.367611684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:35512"
level=warn ts=2026-02-16T11:32:17.469287208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:32:22.469305743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:32:27.468467419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:32:27.468494933Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:32:27.468482969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:32:27.468503856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:32:29.024979262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=warn ts=2026-02-16T11:32:32.46908112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:32:32.919488959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:32:37.468790658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:32:38.780375105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:32:42.468509291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:32:47.469253539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:32:52.469314371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:32:57.452464384Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:32:57.453510813Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:32:57.468731573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:32:57.468762604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:32:57.468771534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:32:57.468777791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:32:58.852143837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:32:59.028560875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=warn ts=2026-02-16T11:33:02.468916445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:33:04.446388813Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:33:07.468423575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:33:12.469124246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:33:17.469202629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:33:22.468499539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:33:27.308567447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:33:27.310451417Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:33:27.468904564Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:33:27.468934539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:33:27.468925736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:33:27.46894292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:33:29.032090482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:33:29.301457807Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:33:29.301525601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:33:29.310471535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:33:32.468768207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:33:37.469104957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:33:42.030330663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:33:42.043261938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:33:42.468715706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:33:47.469239438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:33:52.46890778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:33:57.453604601Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:33:57.454552584Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:33:57.468726669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:33:57.468747926Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:33:57.468741958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:33:57.468756581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:33:59.035070813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:34:01.916712776Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:34:02.468638716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:34:07.468582717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:34:07.488742311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:34:07.49501618Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:34:11.245202416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:34:12.468719609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:34:15.375889158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:47468"
ts=2026-02-16T11:34:15.76467498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45186"
level=warn ts=2026-02-16T11:34:17.469063988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:34:22.469140874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:34:27.469233554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:34:27.469233661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:34:27.469262074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:34:27.469270898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:34:29.038281067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:34:29.450262427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:34:32.468474401Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:34:37.469158862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:34:42.468955749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:34:45.378478114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45456"
level=warn ts=2026-02-16T11:34:47.468466271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:34:52.46893385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:34:57.452519195Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:34:57.453579109Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:34:57.468822895Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:34:57.468856296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:34:57.468867304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:34:57.468865775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:34:58.717974331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:34:58.724312018Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T11:34:59.041230946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:35:01.805355154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:35:02.46872528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:35:07.469137617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:35:07.956657368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:35:12.46919216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:35:17.469206439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:35:22.468981147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:35:27.468567583Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:35:27.46860272Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:35:27.468613047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:35:27.468610313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:35:29.044069291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:35:30.14252323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33078"
level=warn ts=2026-02-16T11:35:32.469219257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:35:33.529156472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:35:37.468710101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:35:42.468433991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:35:45.383455043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59298"
level=warn ts=2026-02-16T11:35:47.469183963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T11:35:52.469137034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:35:57.452779568Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:35:57.453090789Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:35:57.468359462Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:35:57.468388829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:35:57.468396718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:35:57.468404169Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T11:35:57.468444841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:35:57.468548135Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-02-16T11:35:57.468737636Z caller=checkpoint.go:378 msg="writing series" size="512 B"
level=info ts=2026-02-16T11:35:58.174859536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:35:58.182029596Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:35:58.21060403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:35:59.046639292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:36:01.969115506Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T11:36:02.468749962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:06.226682536Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:36:06.468876711Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-02-16T11:36:07.469146628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:10.969348788Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-02-16T11:36:12.469055986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:36:15.386332359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:57022"
level=debug ts=2026-02-16T11:36:15.469186902Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T11:36:17.46913107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:19.969591843Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-02-16T11:36:22.468952042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:24.469391111Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T11:36:27.469177829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:36:27.469207615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:36:27.46921577Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:36:27.469235173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:28.969238825Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-02-16T11:36:29.049835324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:36:30.146827153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50810"
level=debug ts=2026-02-16T11:36:30.756992076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:36:30.757049402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="26 kB" avg_uncomp="26 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:36:30.764755397Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:36:31.057544171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="47 kB" avg_comp="47 kB" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:36:31.061500296Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:36:31.357218163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:36:31.360952347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:36:31.657304211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:36:31.661061231Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T11:36:32.468471553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:33.468801269Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T11:36:37.469270503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:37.968925114Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
level=warn ts=2026-02-16T11:36:42.468476821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:42.469570398Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-02-16T11:36:43.27843306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:36:43.283517974Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-02-16T11:36:45.389224131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:47960"
level=info ts=2026-02-16T11:36:46.20017192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:36:46.203538743Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T11:36:46.969305839Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T11:36:47.468535374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:36:50.643835992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:36:50.647477916Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:36:50.886094959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:36:50.892877804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:36:51.445687857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:36:51.449228281Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:36:51.469591209Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-02-16T11:36:52.46849456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:36:55.969197475Z caller=checkpoint.go:378 msg="writing series" size="381 kB"
level=debug ts=2026-02-16T11:36:57.45259966Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:36:57.453642365Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:36:57.468922572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:36:57.468952448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:36:57.468960593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:36:57.468990799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:36:59.053007321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T11:36:59.304025177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:36:59.30408252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T11:36:59.307347355Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:36:59.604336554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="56 kB" avg_uncomp="56 kB" idle=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:36:59.607611192Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:36:59.904526289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:36:59.90769335Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:37:00.204436949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:37:00.20766125Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:37:00.469623873Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T11:37:00.504762871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:37:00.50808884Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:37:00.804790132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:37:00.809859111Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:37:01.104560807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="3.1 kB" avg_comp="3.1 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:37:01.108215119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:37:01.405106088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="5.0 kB" avg_comp="5.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:37:01.408365717Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:37:01.704435172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="14 kB" avg_uncomp="14 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:37:01.70802149Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:37:02.468361773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:37:04.969290567Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-02-16T11:37:07.468308347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:37:09.468871238Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=debug ts=2026-02-16T11:37:09.468897882Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=17
level=info ts=2026-02-16T11:37:11.148989664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:37:11.152588904Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:37:12.468371464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:13.61185405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:37:13.615047343Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:37:13.969249404Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=info ts=2026-02-16T11:37:14.835849572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:37:14.839447098Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:37:14.843604011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:37:14.846658178Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:37:15.181588732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:37:15.185115637Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:37:15.337469749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:37:15.341165996Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-02-16T11:37:15.391913389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:39010"
ts=2026-02-16T11:37:15.778664623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58290"
level=debug ts=2026-02-16T11:37:15.783107254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T11:37:16.785363765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:37:16.78922902Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T11:37:17.468602621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:17.485732765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:37:17.489301353Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:37:17.506349175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:37:17.509515999Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T11:37:18.469252917Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=info ts=2026-02-16T11:37:18.723866062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:37:18.727231792Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:37:19.59529734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:37:19.599119729Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:37:21.010377161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:37:21.013565512Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:37:22.3322199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:37:22.335565678Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T11:37:22.468794598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:37:22.969770499Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T11:37:27.468244045Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:37:27.468274571Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:37:27.468283564Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:37:27.468318512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:37:27.469299105Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-02-16T11:37:28.57269861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:37:28.572761028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="7.5 kB" avg_comp="7.5 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" idle=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:37:28.5769971Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:37:28.872944971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="96 kB" avg_comp="96 kB" total_uncomp="96 kB" avg_uncomp="96 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:37:28.877035511Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T11:37:29.055748644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:37:29.17348548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:37:29.178853823Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:37:31.96972824Z caller=checkpoint.go:378 msg="writing series" size="133 kB"
level=warn ts=2026-02-16T11:37:32.469330723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:33.631658989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:37:33.63941391Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:37:36.468866974Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T11:37:37.283853933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:37:37.287124065Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:37:37.468938436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:37.789554521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:37:37.793194757Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:37:38.307228151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:37:38.310416938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:37:39.898853251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:37:39.90258998Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:37:40.292711245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:37:40.296637948Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T11:37:40.969110683Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-02-16T11:37:42.468416845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:44.538092362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:37:44.541894464Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:37:45.469577393Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-02-16T11:37:45.800351436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:37:45.80368578Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:37:46.365170795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:37:46.368308466Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:37:47.200393405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:37:47.203515591Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:37:47.468706485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:47.815568172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:37:47.818693608Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:37:49.972185091Z caller=checkpoint.go:378 msg="writing series" size="4.0 MB"
level=debug ts=2026-02-16T11:37:49.972210372Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.6 MB" series=9
level=warn ts=2026-02-16T11:37:52.46875315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:37:52.814353304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:37:52.817768422Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:37:53.166249498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:37:53.169444669Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T11:37:54.468873655Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=debug ts=2026-02-16T11:37:54.533018751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:37:57.452909008Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:37:57.452909598Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:37:57.46824446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:37:57.468265854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:37:57.468273302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:37:57.468296889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:37:58.968895443Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
ts=2026-02-16T11:37:59.058403914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=warn ts=2026-02-16T11:38:02.468436553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:03.142719308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:38:03.146961338Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:38:03.469088035Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T11:38:03.859012209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:38:03.864885403Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:38:07.468602273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:38:07.969736561Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-02-16T11:38:08.332698464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:38:08.33620452Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:38:08.730478044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:38:08.734166299Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:38:10.656352859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:38:10.660065394Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:38:12.469250006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:38:12.470124982Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-02-16T11:38:12.470140487Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-02-16T11:38:13.645686192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:38:13.649266386Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T11:38:16.968832306Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T11:38:17.42737174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:38:17.430878589Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T11:38:17.468675415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:19.26234862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:38:19.269195461Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:38:19.684489431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:38:19.687724184Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T11:38:21.469457963Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-02-16T11:38:22.468782647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:22.844095521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:38:22.847579237Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:38:25.969314944Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T11:38:26.752777274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:38:26.752852234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="23 kB" avg_uncomp="23 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:38:26.756519815Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:38:27.469115582Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:38:27.469141968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:38:27.469149572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:38:27.469165152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:38:29.061927735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:38:29.106979987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:38:29.108648848Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:38:30.469756408Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T11:38:32.468823908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:33.248153697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:38:33.251319968Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:38:33.560916176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:38:33.564114183Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:38:34.969664129Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-02-16T11:38:35.25657327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:38:35.261112744Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:38:36.950178463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:38:36.953763566Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:38:36.967841327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:38:36.970827922Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:38:37.468414045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:39.427663737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:38:39.431004Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:38:39.469472997Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-02-16T11:38:41.982332103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:38:41.985551423Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:38:42.468758641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:43.167272016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:38:43.170404578Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:38:43.969583034Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=info ts=2026-02-16T11:38:45.344585076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:38:45.348204769Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:38:47.468614153Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:38:47.924650446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:38:47.928192534Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:38:48.469035477Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=info ts=2026-02-16T11:38:51.742733661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:38:51.746028271Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T11:38:52.469016876Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:38:52.969023641Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T11:38:55.016738312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:38:55.020281189Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T11:38:57.452576086Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:38:57.453635044Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:38:57.468958719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:38:57.46897976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:38:57.468988548Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:38:57.469017423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:38:57.46905725Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-02-16T11:38:59.046902354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:38:59.05132948Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T11:38:59.064517456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:38:59.986649089Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:38:59.986713036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="49 kB" avg_comp="49 kB" total_uncomp="340 kB" avg_uncomp="340 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:38:59.990313656Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T11:39:00.156761216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33358"
level=debug ts=2026-02-16T11:39:01.968997873Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=warn ts=2026-02-16T11:39:02.469305983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:02.903686908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:39:02.90742829Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:39:05.133156137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:39:05.136487572Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:39:06.468779596Z caller=checkpoint.go:378 msg="writing series" size="8.3 kB"
level=warn ts=2026-02-16T11:39:07.469261845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:10.142316314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:39:10.147289109Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:39:10.969727782Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-02-16T11:39:11.013165124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:39:11.016725984Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:39:11.391891799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:39:11.396019757Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T11:39:12.469254784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:15.433693863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:39:15.437316116Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T11:39:15.469817612Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
level=info ts=2026-02-16T11:39:17.334780686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:39:17.341261472Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:39:17.469020099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:19.403856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:39:19.40879227Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:39:19.759361688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:39:19.762605797Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T11:39:19.969317044Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-02-16T11:39:21.120079369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:39:21.123441316Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:39:22.469320743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:39:24.469508772Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T11:39:24.73148067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:39:25.985854287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:39:25.989075551Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:39:27.469197194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:39:27.46922807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:39:27.469236632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:39:27.469240652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:28.453455946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:39:28.456856266Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:39:28.96932276Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
ts=2026-02-16T11:39:29.067647334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T11:39:29.400762942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:39:29.405384473Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:39:30.029359152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:39:30.029416353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="294 B" avg_comp="294 B" total_uncomp="294 B" avg_uncomp="294 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:39:30.032624338Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:39:32.468862969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:33.354012246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:39:33.357415582Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T11:39:33.469000506Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T11:39:36.08448408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:39:36.088090367Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:39:37.468796056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:37.962495234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:39:37.966226212Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:39:37.969309949Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-02-16T11:39:39.410315037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:39:39.413658003Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:39:39.815944993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:39:39.819707991Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:39:39.860067536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:39:39.863037872Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:39:41.425543806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:39:41.429452831Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:39:41.931689988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:39:41.934966551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:39:42.468705847Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:39:42.468982469Z caller=checkpoint.go:378 msg="writing series" size="359 kB"
level=debug ts=2026-02-16T11:39:42.468998519Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=20
ts=2026-02-16T11:39:45.788369997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40444"
level=debug ts=2026-02-16T11:39:46.969159758Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=warn ts=2026-02-16T11:39:47.468337213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:39:48.885002003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:39:48.888711976Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:39:51.469199755Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T11:39:52.12802387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:39:52.1319862Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T11:39:52.469112852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:39:55.969007746Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T11:39:57.376981014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:39:57.380751461Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:39:57.452918361Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:39:57.452924443Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:39:57.468265297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:39:57.468286215Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:39:57.468294161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:39:57.468317868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:39:59.070322345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:39:59.77430256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:39:59.778210625Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:40:00.056221754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T11:40:00.16119143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38356"
level=debug ts=2026-02-16T11:40:00.469585992Z caller=checkpoint.go:378 msg="writing series" size="5.6 kB"
level=info ts=2026-02-16T11:40:01.815288559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:40:01.818604271Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:40:02.46912369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:40:04.969711142Z caller=checkpoint.go:378 msg="writing series" size="595 kB"
level=info ts=2026-02-16T11:40:05.43471598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:40:05.438230736Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:40:06.746178827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:40:06.752683896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:40:06.752704335Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:40:06.752725725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:40:06.758399086Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:40:07.468601984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:40:09.469150686Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-02-16T11:40:12.164849201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:40:12.170529468Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:40:12.298941888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:40:12.302339564Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:40:12.469216274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:12.790927081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:40:12.794100931Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:40:13.968826823Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-02-16T11:40:14.170338247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:40:14.173610772Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T11:40:15.401187711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:53436"
level=info ts=2026-02-16T11:40:16.879691937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:40:16.883294321Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:40:17.468343594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:40:18.469029Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T11:40:18.774072922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:40:18.777056965Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:40:20.740148351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:40:20.777411575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:40:20.780965208Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:40:21.529592946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:40:21.533117428Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:40:22.469131913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:22.671684055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:40:22.676546208Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T11:40:22.969027323Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:40:25.119820495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:40:25.126958552Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:40:26.001915732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:40:26.005294854Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:40:26.696064275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:40:26.699360065Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:40:27.468679511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:40:27.468708981Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:40:27.468696457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:27.468717279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T11:40:27.468736457Z caller=checkpoint.go:390 msg="flushing series" totalSize="831 kB" series=9
level=info ts=2026-02-16T11:40:27.476154295Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-02-16T11:40:27.479651704Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010961469s
level=debug ts=2026-02-16T11:40:27.675121315Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T11:40:29.072952255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=warn ts=2026-02-16T11:40:32.469049884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:34.874866936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:40:34.879886627Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:40:36.051654421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:40:36.055184361Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:40:37.468344944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:39.383459578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:40:39.38711091Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:40:40.229319552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:40:40.232932303Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:40:41.33822939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:40:41.341645689Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:40:41.885309179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:40:41.888531693Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:40:42.4683713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:40:45.793234907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52720"
level=info ts=2026-02-16T11:40:46.169935487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:40:46.176232229Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:40:47.468537333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:49.362921591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:40:49.366360927Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:40:50.821080781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:40:50.824639743Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:40:52.468557751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:53.405185031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:40:53.409800769Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:40:57.452776232Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:40:57.453162749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:40:57.468429862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:40:57.468444669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:40:57.468452426Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:40:57.468465294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:40:57.468479371Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T11:40:57.468629972Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-02-16T11:40:57.468829348Z caller=checkpoint.go:378 msg="writing series" size="518 B"
ts=2026-02-16T11:40:59.075544473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:40:59.849326852Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:40:59.849386007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="190 B" avg_comp="190 B" total_uncomp="60 kB" avg_uncomp="60 kB" idle=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:40:59.853405043Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:41:01.823965941Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T11:41:02.468915148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:03.394082694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:41:03.39778403Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:41:05.063090061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:41:05.067108934Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:41:05.461905982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:41:05.465267679Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T11:41:06.179094795Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-02-16T11:41:07.468494021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:09.021012155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:41:09.024634948Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:41:10.533876969Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=info ts=2026-02-16T11:41:10.689421354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:41:10.694252087Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:41:12.468989558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:41:14.888849798Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
ts=2026-02-16T11:41:15.796235962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50986"
level=info ts=2026-02-16T11:41:16.883801292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:41:16.88713653Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:41:17.46892144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:17.614244609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:41:17.617534185Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:41:18.721917505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:41:18.845254273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:41:18.848584932Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:41:19.243436546Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T11:41:21.24782182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:41:21.251273052Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:41:22.468818026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:22.576113321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:41:22.579565121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:41:22.808921918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:41:22.812112841Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:41:23.212973508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:41:23.216351215Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:41:23.598341244Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T11:41:24.187853631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:41:24.19253212Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:41:25.018042532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:41:25.021485227Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T11:41:26.994376108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:41:27.46865453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:41:27.468688744Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:41:27.468697371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:41:27.468692715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:41:27.95334506Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T11:41:29.077954249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:41:30.310120383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:41:30.316422598Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:41:31.589327773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:41:31.593129664Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:41:32.308478679Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T11:41:32.385759036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:41:32.389405342Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T11:41:32.469059025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:35.264237291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:41:35.269294306Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T11:41:36.663222087Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-02-16T11:41:37.468768973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:38.624793792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:41:38.628097011Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T11:41:41.017936705Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T11:41:42.468558737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:43.416584586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:41:43.420292321Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:41:45.372947763Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
ts=2026-02-16T11:41:45.407318237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:41104"
level=warn ts=2026-02-16T11:41:47.46928821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:49.700356295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:41:49.707082402Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-02-16T11:41:49.727177892Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-02-16T11:41:52.329019491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:41:52.33266378Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:41:52.468719187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:41:53.634581066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:41:53.637856708Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:41:53.709210183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:41:53.712392741Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:41:54.082596716Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=info ts=2026-02-16T11:41:55.662084671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:41:55.665939257Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:41:56.775171051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:41:56.778641298Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:41:57.452893567Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:41:57.452901158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:41:57.469246944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:41:57.46927573Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:41:57.469284577Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:41:57.46930053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:41:58.437140834Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-02-16T11:41:59.080353447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:41:59.213029187Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:42:02.468339776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:42:02.792195013Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T11:42:03.25042631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:42:03.253980454Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:42:05.785076431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:42:05.788800014Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:42:06.923842135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:42:06.927003764Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:42:07.146911247Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-02-16T11:42:07.468445205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:42:08.159023761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:42:08.162361696Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T11:42:08.725474143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:42:09.353476091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:42:09.35688715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:42:11.501737749Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-02-16T11:42:11.50176051Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=18
level=info ts=2026-02-16T11:42:11.601854296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:42:11.6081288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:42:11.702419873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:42:11.70604797Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T11:42:12.46925856Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:42:13.292840164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:42:13.295935433Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:42:15.330132994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:42:15.333522859Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:42:15.406723067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:42:15.40995272Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:42:15.856398806Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=warn ts=2026-02-16T11:42:17.469203879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:42:20.033545146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:42:20.037508411Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:42:20.211276539Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=info ts=2026-02-16T11:42:20.987405975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:42:20.991498265Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:42:21.297710089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:42:21.301872758Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T11:42:22.468612759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:42:24.566356737Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T11:42:27.468245381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:42:27.468272578Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:42:27.468280483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:42:27.468328666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:42:27.582580872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:42:27.587691492Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T11:42:28.921303035Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T11:42:29.083079623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T11:42:30.172011257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60924"
level=info ts=2026-02-16T11:42:32.023929571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:42:32.027886434Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:42:32.468914833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:42:32.827634644Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:42:33.276033133Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=info ts=2026-02-16T11:42:35.737185248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:42:35.740993014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:42:37.468465732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:42:37.630996471Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T11:42:38.713387929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:42:38.718169606Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:42:41.000859908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:42:41.004636647Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:42:41.985939933Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=warn ts=2026-02-16T11:42:42.468753499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:42:43.814117966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:42:43.817471895Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:42:44.149580438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:42:44.153278842Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T11:42:45.412496828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:50686"
level=info ts=2026-02-16T11:42:45.922280619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:42:45.927018556Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:42:46.340448802Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=info ts=2026-02-16T11:42:46.906399425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:42:46.91021033Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T11:42:47.468311256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:42:49.433345856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:42:49.434989376Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:42:50.698476573Z caller=checkpoint.go:378 msg="writing series" size="4.3 MB"
level=debug ts=2026-02-16T11:42:50.698500604Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=9
level=info ts=2026-02-16T11:42:51.074519807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:42:51.078347007Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:42:52.469181762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:42:54.063209396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:42:54.069681074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:42:55.050732575Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=debug ts=2026-02-16T11:42:57.452916645Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:42:57.452917077Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:42:57.468292148Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:42:57.46830175Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:42:57.468316093Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:42:57.468326376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:42:59.086593736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:42:59.404799308Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-02-16T11:43:00.767034939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:43:00.770607206Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:43:01.412090289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:43:01.415598924Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:43:01.415619368Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:01.51901881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:43:01.522772136Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:43:01.610013094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:43:01.613407737Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:43:01.715736295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:43:01.719102887Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:43:01.914700701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:43:01.918048606Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:43:01.91807078Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:02.016309169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:43:02.019608692Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T11:43:02.468610802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:03.04785115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:43:03.05126998Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:43:03.759570514Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T11:43:04.184741861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:43:04.187898879Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T11:43:04.18791695Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:04.187937732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:43:04.190672913Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:43:06.873004501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:43:06.946230373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:43:06.94960678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T11:43:06.9496274Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:06.949648469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:43:06.952495846Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T11:43:07.468558605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:43:08.114853278Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T11:43:09.421002412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:43:09.424179679Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:43:09.424197541Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:09.42421808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="219 B" avg_comp="219 B" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" idle=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:43:09.426763541Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:43:12.308024413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:43:12.311254049Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:43:12.469115237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:43:12.469836838Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T11:43:12.469852827Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-02-16T11:43:15.600106092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:43:15.603800751Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T11:43:16.824718504Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-02-16T11:43:17.469040242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:17.922948751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:43:17.92710801Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:43:18.942040265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:43:18.945988564Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:43:18.946009338Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:18.946030826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:43:18.948946895Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:43:19.585005085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:43:19.588489545Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:43:19.769181642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:43:19.772604969Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T11:43:19.77262547Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:19.772646243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:43:19.77556782Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:43:20.825087354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:43:20.828333418Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:43:20.828351114Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:20.828372054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:43:20.831360107Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:43:21.178761431Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-02-16T11:43:22.469087907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:23.094518226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:43:23.098339526Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:43:25.53436507Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T11:43:27.039044773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:43:27.042414982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:43:27.468369801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:43:27.46838318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:43:27.468399701Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:43:27.468407925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:43:28.737944583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:43:28.741228983Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:43:28.741248038Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:28.741267985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:43:28.743868082Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T11:43:29.089750366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:43:29.888628716Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-02-16T11:43:31.639708296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:43:31.643630702Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:43:32.469267469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:33.304834131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:43:33.308449828Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:43:33.308474735Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:33.30851628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:43:33.311418583Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:43:33.748631248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:43:34.243896573Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-02-16T11:43:36.133954814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:43:36.137534804Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:43:36.576003134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:43:36.5793942Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:43:36.977555915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:43:36.980865923Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:43:37.468491438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:37.551858911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:43:37.555796782Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:43:37.555836811Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:37.555863017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:43:37.558557107Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:43:38.330119701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:43:38.333345215Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:43:38.598380301Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-02-16T11:43:39.94026111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:43:39.944590767Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:43:40.222238624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:43:40.22540545Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:43:40.495487587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:43:40.502404119Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:43:41.030723956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:43:41.033939919Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T11:43:42.469109508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:42.49018275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:43:42.493558268Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:43:42.493575782Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:42.493595656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:43:42.496223684Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T11:43:42.953083211Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
ts=2026-02-16T11:43:45.416602172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55826"
level=info ts=2026-02-16T11:43:45.624779146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:43:45.628169107Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:43:45.628187618Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:45.628207748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:43:45.630884801Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T11:43:45.808545016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40296"
level=info ts=2026-02-16T11:43:46.059637758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:43:46.063090368Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:43:47.307710948Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T11:43:47.468902926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:43:50.327960343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:43:50.38118091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:43:50.38424878Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:43:51.575928132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:43:51.579063041Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:43:51.579091633Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:51.579115228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:43:51.581634418Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:43:51.663022346Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T11:43:52.46903477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:54.325924393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:43:54.329977573Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:43:54.846723081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:43:54.850115315Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:43:55.372612065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:43:55.376125108Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:43:55.623825019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:43:55.627079295Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:43:56.017741587Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-02-16T11:43:57.122922247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:43:57.126249515Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:43:57.1262684Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:43:57.126291136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:43:57.129236374Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:43:57.452768248Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:43:57.45316394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:43:57.46842955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:43:57.468448496Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:43:57.468442421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:43:57.468457361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:43:59.092907215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:44:00.074767278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:44:00.08162085Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T11:44:00.372656716Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=info ts=2026-02-16T11:44:01.040058902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:44:01.043857563Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:44:01.70766282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:44:01.7111374Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:44:01.711162827Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:44:01.711192625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:44:01.714074791Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:44:01.878679239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:44:01.882612178Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T11:44:02.46851506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:04.727212845Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T11:44:04.825250086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:44:04.828386138Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:44:05.312871553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.4 kB" avg_comp="2.4 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:44:05.316354397Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:44:05.584295379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:44:05.587339388Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:44:07.469256384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:07.677733281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:44:09.082117632Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T11:44:09.706077092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:44:09.709473779Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:44:12.240475655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:44:12.244416532Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:44:12.412848962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:44:12.416259922Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:44:12.468702966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:13.437244005Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T11:44:15.013344262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:44:15.016735382Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T11:44:15.811279541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60450"
level=warn ts=2026-02-16T11:44:17.468631424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:17.792171464Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:44:19.062211454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:44:19.06629147Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:44:21.229898708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:44:21.234277285Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:44:21.877229028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:44:21.88095228Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:44:22.146929761Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-02-16T11:44:22.468485405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:44:24.342351683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:44:24.346213958Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:44:25.744165583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:44:25.747886011Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T11:44:26.502052784Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=debug ts=2026-02-16T11:44:26.502079251Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=17
level=info ts=2026-02-16T11:44:27.468240229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:44:27.468267707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:44:27.468276589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:44:27.468303158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:44:28.417231778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:44:28.420511401Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T11:44:29.095587556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T11:44:30.179961081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33194"
level=info ts=2026-02-16T11:44:30.330676049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:44:30.334146604Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:44:30.401314136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:44:30.407628332Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:44:30.855963044Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T11:44:31.54627592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:44:31.549739801Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:44:32.468508954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:34.184525677Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:44:35.211714348Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-02-16T11:44:37.468814926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:44:38.665002646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:44:38.66860548Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:44:38.668627342Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:44:38.668646351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:44:38.671458888Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:44:39.513923221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:44:39.51725331Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:44:39.567320536Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T11:44:39.567343111Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-02-16T11:44:40.155606973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:44:41.314968606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:44:41.318354656Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:44:42.469351607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:43.921161556Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-02-16T11:44:44.056665791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:44:44.064464162Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T11:44:45.4197847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:42882"
level=info ts=2026-02-16T11:44:47.407286174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:44:47.410882458Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:44:47.468479999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:48.276003177Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T11:44:49.447602381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:44:49.452405668Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:44:50.515786158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:44:50.519180124Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:44:50.64663926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:44:50.650692124Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:44:50.682835699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:44:50.685883539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:44:50.800160883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:44:50.80349453Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:44:51.165244738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:44:51.168826016Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:44:52.468606023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:44:52.630735765Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=debug ts=2026-02-16T11:44:56.985276643Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=debug ts=2026-02-16T11:44:57.452692118Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:44:57.453754772Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:44:57.469053758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:44:57.469082232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:44:57.469093535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:44:57.469086233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:44:58.731035923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:44:58.735257316Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T11:44:59.098533391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:45:00.183044621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35844"
level=debug ts=2026-02-16T11:45:01.34043361Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=warn ts=2026-02-16T11:45:02.469066106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:04.493847605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:45:04.497347036Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:45:04.525413688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:45:04.528484407Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:45:05.127228914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:45:05.13077238Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:45:05.307271243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:45:05.310546305Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:45:05.694723013Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=info ts=2026-02-16T11:45:07.040649603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:45:07.043932287Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:45:07.468916075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:07.971012099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:45:07.974409087Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:45:08.162006327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:45:09.081759418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="96 kB" avg_uncomp="96 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:45:09.085267424Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T11:45:10.050547651Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T11:45:11.695098191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:45:11.702712942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:45:12.167928561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:45:12.171463046Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:45:12.469077001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:45:14.404942791Z caller=checkpoint.go:378 msg="writing series" size="830 B"
level=info ts=2026-02-16T11:45:15.608572211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:45:15.612055124Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T11:45:15.816338232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52088"
level=info ts=2026-02-16T11:45:17.457972394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:45:17.461706593Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:45:17.468825065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:45:18.760068755Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T11:45:22.062394196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:45:22.065952195Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T11:45:22.468539042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:45:23.114503123Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T11:45:27.469235605Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:45:27.469267227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:45:27.469276325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T11:45:27.469268958Z caller=checkpoint.go:390 msg="flushing series" totalSize="645 kB" series=10
level=warn ts=2026-02-16T11:45:27.469310546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:27.564166994Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-02-16T11:45:27.565660876Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.096888467s
ts=2026-02-16T11:45:29.101519017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:45:30.183942402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:45:30.188561432Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:45:31.054312426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:45:31.057836819Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:45:32.468514086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:34.534032934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:45:34.538386498Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T11:45:36.023731558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:45:36.023786711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="350 B" avg_uncomp="350 B" idle=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:45:36.027643159Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T11:45:37.468353941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:38.587747966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" idle=1 full=1 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:45:38.591186737Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:45:39.502600209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:45:39.506066974Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:45:39.506087914Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:45:39.50610917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:45:39.508865666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:45:40.630546781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:45:40.634357913Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:45:41.364602934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T11:45:42.468939994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:45:45.424725797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:58762"
ts=2026-02-16T11:45:45.818924345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53346"
level=info ts=2026-02-16T11:45:46.350680548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:45:46.354403152Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:45:47.318262797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" idle=1 full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:45:47.321935275Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T11:45:47.468642022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:48.471215769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:45:48.474712793Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:45:49.631101765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:45:49.635279779Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:45:51.507274833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" idle=1 full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:45:51.512148356Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:45:52.223330212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="2.5 kB" avg_comp="1.2 kB" total_uncomp="52 kB" avg_uncomp="26 kB" idle=1 full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:45:52.226964285Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:45:52.468741758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:45:52.685026883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:45:52.691505969Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:45:52.691527853Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:45:52.691549749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:45:52.698089912Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:45:56.015445228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:45:56.018763297Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:45:56.622782124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:45:56.626139985Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:45:57.452793493Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:45:57.4533071Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T11:45:57.454565265Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T11:45:57.454606257Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968046 n=1
level=debug ts=2026-02-16T11:45:57.454642805Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968046 n=1
level=info ts=2026-02-16T11:45:57.468545317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:45:57.468557001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:45:57.468568977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:45:57.4685901Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:45:57.46859157Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T11:45:57.468689255Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-02-16T11:45:57.468843981Z caller=checkpoint.go:378 msg="writing series" size="518 B"
level=info ts=2026-02-16T11:45:57.917529756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:45:57.921065793Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:45:57.921086946Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:45:57.921109474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:45:57.923769004Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T11:45:59.104707476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:46:00.188311742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50904"
level=debug ts=2026-02-16T11:46:01.824078832Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:46:02.216361257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:46:02.220162091Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:46:02.469251365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:03.675719507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:46:03.679259289Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:46:04.052786543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:46:04.057154885Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:46:05.058526088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:46:05.06186417Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:46:06.178789163Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-02-16T11:46:06.522828013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:46:06.526162051Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:46:07.468550138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:46:10.447222568Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:46:10.533459647Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-02-16T11:46:10.717120605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:46:10.724178657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:46:11.139859992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:46:11.143656769Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:46:11.715290933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:46:11.718483577Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:46:12.469087284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:13.154039236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:46:13.157523731Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:46:14.872355838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:46:14.876956322Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:46:14.888867244Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
ts=2026-02-16T11:46:15.428105191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40358"
level=info ts=2026-02-16T11:46:16.426328676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:46:16.42968095Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:46:17.469009418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:46:19.24408883Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T11:46:19.30337453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:46:19.306543642Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:46:22.469114092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:46:23.598335001Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T11:46:26.846614739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:46:26.852048707Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:46:27.469238237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:46:27.469267292Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:46:27.469282178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:46:27.469289244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:46:27.953430619Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T11:46:29.107659805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:46:29.63166777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:46:29.636035306Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T11:46:30.191075964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40520"
level=info ts=2026-02-16T11:46:31.575750855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:46:31.579271332Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:46:31.799934733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:46:31.803181759Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:46:32.307701866Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T11:46:32.430050052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:46:32.43491505Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T11:46:32.468868344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:34.632426078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:46:34.636990142Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:46:35.374473906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:46:35.378046486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:46:36.663053358Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=warn ts=2026-02-16T11:46:37.469144065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:38.597523767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="2.5 kB" avg_comp="1.2 kB" total_uncomp="52 kB" avg_uncomp="26 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:46:38.601135079Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:46:39.923169255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:46:39.926202047Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T11:46:41.018304238Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-02-16T11:46:41.922852512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:46:41.926047245Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:46:41.948235941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T11:46:42.469220852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:43.937325039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:46:43.945474126Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:46:45.373085325Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
ts=2026-02-16T11:46:45.823259165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57852"
level=debug ts=2026-02-16T11:46:45.89206186Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:46:47.018854509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:46:47.022279543Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:46:47.468328548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:47.839186573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:46:47.843347962Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:46:48.786260094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:46:48.789544355Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:46:49.72748516Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-02-16T11:46:50.104721944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:46:50.108232501Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T11:46:52.468590646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:46:54.082372034Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=info ts=2026-02-16T11:46:55.94639069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:46:55.950852543Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:46:57.452775085Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:46:57.453235475Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:46:57.468505691Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:46:57.468538399Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:46:57.468549298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:46:57.468576792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:46:57.828985151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:46:57.834736969Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:46:58.436784323Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T11:46:58.436827413Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=15
ts=2026-02-16T11:46:59.110088306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:46:59.20263174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:46:59.20610905Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:47:02.468942526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:47:02.791478293Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-02-16T11:47:07.146864715Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-02-16T11:47:07.468432087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:47:07.71302773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:47:07.717273774Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:47:08.536569015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:47:08.540626933Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:47:10.287339194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:47:10.291280758Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T11:47:11.501514986Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-02-16T11:47:12.136243792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:47:12.140500617Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:47:12.468835722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:47:15.433488722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:38770"
level=info ts=2026-02-16T11:47:15.45205987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:47:15.455858007Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:47:15.856290661Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-02-16T11:47:16.223878223Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:47:17.46924581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:47:18.030599345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:47:18.0349939Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:47:18.038025022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:47:18.041514438Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:47:19.198247194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:47:19.201321665Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:47:20.211048257Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=info ts=2026-02-16T11:47:21.868090643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="2.5 kB" avg_comp="1.2 kB" total_uncomp="52 kB" avg_uncomp="26 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:47:21.871589685Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:47:22.468791854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:47:23.257871646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:47:23.261622596Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:47:24.566276724Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T11:47:24.736663024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:47:24.740334354Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:47:26.193016423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:47:26.199841809Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:47:27.468513974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:47:27.468547502Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:47:27.468556459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:47:27.468606314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:47:28.920829912Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T11:47:29.112528971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T11:47:30.164496089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:47:30.167795079Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:47:30.205591741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:47:30.208840837Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:47:31.589204136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:47:31.592603075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:47:32.468344837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:47:32.623116423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:47:32.624775969Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:47:33.27560007Z caller=checkpoint.go:378 msg="writing series" size="271 kB"
level=debug ts=2026-02-16T11:47:33.275624105Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-02-16T11:47:37.468330834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:47:37.630739033Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T11:47:39.527346004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:47:39.531784534Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:47:39.531829748Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:47:39.531853848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:47:39.535155282Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:47:41.986014906Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=warn ts=2026-02-16T11:47:42.468939956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:47:45.934502208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:47:45.939424019Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T11:47:46.00762966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:47:46.011148748Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:47:46.135438129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:47:46.139033266Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:47:46.272848104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:47:46.32385331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:47:46.327510305Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:47:46.340620074Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-02-16T11:47:47.032697176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:47:47.036557487Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:47:47.468643762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:47:50.698856517Z caller=checkpoint.go:378 msg="writing series" size="5.2 MB"
level=debug ts=2026-02-16T11:47:50.698900168Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.3 MB" series=4
level=info ts=2026-02-16T11:47:50.739556817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:47:50.743783777Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:47:52.468849558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:47:55.049828678Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=info ts=2026-02-16T11:47:55.314380035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:47:55.31764589Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:47:56.712498978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:47:56.716306749Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T11:47:57.452741146Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:47:57.452769928Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:47:57.46910306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:47:57.46912488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:47:57.46911711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:47:57.469135442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:47:58.034296388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:47:58.038121995Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T11:47:59.126057395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:47:59.405295696Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
ts=2026-02-16T11:48:00.197396085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53078"
level=info ts=2026-02-16T11:48:02.137777701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:48:02.143268244Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:48:02.468541126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:48:02.788862303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:48:02.792314056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:48:03.757304058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:48:03.759265663Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=error ts=2026-02-16T11:48:03.760662071Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:48:06.928026171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:48:06.931461177Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T11:48:07.468407193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:48:08.114775654Z caller=checkpoint.go:378 msg="writing series" size="7.6 kB"
level=info ts=2026-02-16T11:48:10.429712893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:48:10.434364511Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:48:12.180399327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:48:12.183871339Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:48:12.46016499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:48:12.463928225Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T11:48:12.469036087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:48:12.469768266Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-02-16T11:48:12.469780813Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-02-16T11:48:12.60832176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:48:12.611567032Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:48:16.613310054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:48:16.617040273Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:48:16.824592943Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-02-16T11:48:17.468484656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:48:17.636931622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:48:17.645963427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:48:17.678721978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:48:17.682210194Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T11:48:17.767514512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:48:21.179194001Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T11:48:21.564034148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="2.5 kB" avg_comp="1.2 kB" total_uncomp="52 kB" avg_uncomp="26 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:48:21.56738837Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=warn ts=2026-02-16T11:48:22.469262554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:48:25.534408579Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T11:48:26.64816521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:48:27.238611921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:48:27.242422783Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:48:27.46898708Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:48:27.469016445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:48:27.469024719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:48:27.469097309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:48:29.128765633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:48:29.888640735Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-02-16T11:48:31.12573303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:48:31.129172052Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:48:32.469056299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:48:33.037847593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:48:33.041795905Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:48:33.741680038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:48:33.745191881Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:48:34.243400069Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-02-16T11:48:35.654511592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:48:35.657923393Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T11:48:37.468738135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:48:37.503911953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:48:37.50717165Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:48:38.598643115Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-02-16T11:48:41.500511008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:48:41.504248717Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T11:48:42.468413574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:48:42.952837703Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=info ts=2026-02-16T11:48:43.743729204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:48:43.74776765Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:48:44.804705513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:48:44.808477597Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:48:45.654714213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:48:45.658439865Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T11:48:45.832929075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42516"
level=info ts=2026-02-16T11:48:46.006376833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:48:46.011506389Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:48:46.359530781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:48:46.363226401Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:48:47.307589719Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T11:48:47.46843307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:48:51.663474768Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T11:48:52.468841118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:48:53.748301592Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:48:56.018024248Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-02-16T11:48:56.970342421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:48:56.975451475Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:48:57.453039649Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:48:57.45327259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:48:57.468549365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:48:57.468574168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:48:57.46858363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:48:57.468647958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:48:58.7122745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:48:58.716254727Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T11:48:59.131454272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:48:59.806858719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:48:59.810490806Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T11:49:00.372328519Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-02-16T11:49:00.637059262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:49:00.640925387Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:49:01.102774566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:49:01.106305108Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T11:49:02.468696545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:49:03.707870463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:49:03.716627873Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:49:04.727560002Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T11:49:06.535156154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:49:06.538413436Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T11:49:07.468991968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:09.082265218Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T11:49:10.692006349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:49:10.697655329Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T11:49:12.468836538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:13.437210199Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T11:49:15.013970619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="2.5 kB" avg_comp="1.2 kB" total_uncomp="52 kB" avg_uncomp="26 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:49:15.018013181Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=info ts=2026-02-16T11:49:16.081936074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:49:16.086289908Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:49:17.469180413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:17.792464284Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:49:18.715936581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:49:18.720056919Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:49:19.241354007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:49:19.245059126Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:49:19.355284574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:49:19.554014098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:49:19.55764947Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:49:21.964293805Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:49:22.146899736Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T11:49:22.309391547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:49:22.312936863Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:49:22.468576816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:49:23.551997272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:49:23.555057496Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:49:26.501941415Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-02-16T11:49:27.145243149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:49:27.148593951Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:49:27.46893682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:49:27.468933796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:27.468964447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:49:27.468973274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:49:29.134460304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:49:30.323399761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:49:30.326753122Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:49:30.85677053Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-02-16T11:49:30.917931705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:49:30.921198946Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T11:49:32.468878985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:49:34.735671256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:49:34.739535058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:49:35.211637573Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=debug ts=2026-02-16T11:49:35.211662799Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=19
level=info ts=2026-02-16T11:49:36.346323043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:49:36.350504014Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c6638380a%3A19c6638380b%3A41460bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T11:49:36.350529177Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:36.350555499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="158 B" avg_comp="158 B" total_uncomp="115 kB" avg_uncomp="115 kB" idle=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:49:36.354221627Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T11:49:37.469327523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:39.567475845Z caller=checkpoint.go:378 msg="writing series" size="3.0 MB"
level=debug ts=2026-02-16T11:49:39.567500123Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=1
level=info ts=2026-02-16T11:49:39.995513791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:49:39.999052598Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:49:39.999072449Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:39.999101756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:49:40.001951717Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:49:40.124411592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.5 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:49:40.127785216Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:49:40.127820457Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:40.427670522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:49:40.430855646Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:49:40.430875046Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:40.599540623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:49:40.602880434Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:49:40.899734024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:49:40.903426787Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:49:42.468546364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:43.920980339Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T11:49:44.581976314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:49:44.585820936Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:49:47.340675508Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:49:47.469499046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:48.276266879Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T11:49:50.494863898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:49:50.499836661Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:49:50.499860692Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:50.800072602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:49:50.80406168Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T11:49:52.469168554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:52.630572562Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T11:49:52.75485472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:49:52.758189972Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:49:54.644490401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:49:54.647873761Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:49:55.999139516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:49:56.003308882Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T11:49:56.506249019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:49:56.512742154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:49:56.985139151Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-02-16T11:49:57.4529483Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:49:57.452981895Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:49:57.468290479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:49:57.468297248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:49:57.468321144Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:49:57.468329682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:49:57.916230009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:49:57.919519637Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T11:49:57.919539718Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:57.919560463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:49:57.922344316Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:49:58.983838991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:49:58.987158964Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T11:49:59.137600813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:49:59.215282732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:49:59.218372962Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:49:59.218392532Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:49:59.218412241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:49:59.221075201Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:50:00.125489285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=2 total_comp="6.6 kB" avg_comp="3.3 kB" total_uncomp="52 kB" avg_uncomp="26 kB" full=2 labels="{stream=\"image-controller\"}"
level=error ts=2026-02-16T11:50:00.128753597Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19c6637d025%3A19c66386604%3A740c58d0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"image-controller\"}"
level=error ts=2026-02-16T11:50:00.12877336Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:00.128793643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="45 kB" avg_uncomp="45 kB" idle=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:50:00.13238826Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T11:50:00.205241096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53412"
level=debug ts=2026-02-16T11:50:00.517666266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:50:00.540741124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:50:00.54397044Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:50:00.543991509Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:00.544016635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:50:00.546935591Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:50:01.185529972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:50:01.189237874Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:50:01.189259556Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:01.189280183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:50:01.192139314Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:50:01.340900649Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=warn ts=2026-02-16T11:50:02.469220219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:04.571373698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:50:04.575110879Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:50:04.575135979Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:04.575160128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:50:04.578444846Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:50:05.486135031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:50:05.49100965Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:50:05.695671626Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-02-16T11:50:05.804035776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:50:05.807710412Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T11:50:05.807734451Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:05.807762184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:50:05.810723547Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:50:07.366720346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:50:07.370691456Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T11:50:07.468902599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:50:10.050412437Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=info ts=2026-02-16T11:50:12.376080246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:50:12.382028256Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:50:12.468507223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:50:12.84505939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:50:12.903834828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:50:12.907114919Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:50:13.798308029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:50:13.80172187Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:50:13.801746314Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:13.80177513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:50:13.804508127Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:50:14.405307895Z caller=checkpoint.go:378 msg="writing series" size="879 B"
level=info ts=2026-02-16T11:50:14.517943669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:50:14.521135198Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:50:15.184083477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:50:15.187502307Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:50:15.21657596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:50:15.219622682Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:50:16.147319242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:50:16.150737358Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:50:16.599477689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:50:16.602745767Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T11:50:17.468774505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:18.399851095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:50:18.404746295Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:50:18.759934195Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T11:50:18.830098282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:50:18.833598668Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T11:50:19.344067397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:50:19.347330737Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:50:21.600319688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:50:21.605408807Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:50:21.605431166Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:21.906382672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:50:21.910000096Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:50:22.46931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:50:23.114483081Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-02-16T11:50:23.190737241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:50:23.194190604Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:50:25.199414109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:50:25.203132833Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:50:25.684542676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:50:25.68897497Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:50:27.468369827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:50:27.468414391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:50:27.468424967Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:50:27.468414056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:50:27.469496709Z caller=checkpoint.go:390 msg="flushing series" totalSize="721 kB" series=10
level=info ts=2026-02-16T11:50:27.477271944Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-02-16T11:50:27.478745273Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00994667s
level=info ts=2026-02-16T11:50:28.027269859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:50:28.03082853Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
ts=2026-02-16T11:50:29.140358024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T11:50:29.754137526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:50:29.757614266Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T11:50:30.207722448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53514"
level=info ts=2026-02-16T11:50:31.683452885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:50:31.6865549Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:50:31.686573681Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:50:31.987313788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:50:31.990513582Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:50:32.468329794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:32.655162792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:50:32.658281246Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:50:34.601342211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:50:34.60493684Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:50:35.923928801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:50:35.927800919Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:50:37.468383096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:37.91817681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:50:37.924372205Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:50:39.80909999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:50:39.812251297Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:50:40.040862367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:50:40.04417784Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:50:41.370289649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:50:41.37352369Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:50:42.361532602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:50:42.3647038Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T11:50:42.469050062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:50:44.424892647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:50:46.049613426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:50:46.760280634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:50:46.763866689Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:50:47.014409419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:50:47.017683636Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T11:50:47.468353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:48.821659677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:50:48.825429047Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:50:52.468988414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:54.218208322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:50:54.222044261Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:50:55.945363554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:50:55.948743321Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:50:56.742771314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" idle=1 full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:50:56.746028649Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:50:57.452784028Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:50:57.453145847Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:50:57.468414721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:50:57.468434199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:50:57.468429201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:50:57.468444606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:50:57.468458115Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T11:50:57.468571419Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-02-16T11:50:57.468724708Z caller=checkpoint.go:378 msg="writing series" size="518 B"
ts=2026-02-16T11:50:59.14294258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:51:01.824618638Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T11:51:02.468918203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:04.956680947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="449 B" avg_comp="449 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:51:04.960270434Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:51:05.007344202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:51:05.010627314Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:51:05.052004637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:51:05.056188241Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:51:05.624652356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:51:05.62812417Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:51:05.628150888Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:51:05.628178749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:51:05.631938706Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:51:06.179086643Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=info ts=2026-02-16T11:51:07.269456958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:51:07.272937666Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:51:07.468898313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:08.072938846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:51:08.076360902Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T11:51:09.327063995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:51:10.533948409Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-02-16T11:51:12.468753767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:51:14.888527161Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-02-16T11:51:15.448672972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:51:15.455346838Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T11:51:15.8448325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47384"
level=info ts=2026-02-16T11:51:16.337951599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:51:16.342557278Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:51:16.578351217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:51:16.582524309Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:51:16.582550307Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:51:16.582579455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:51:16.587138974Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:51:17.46842128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:18.079175195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:51:18.08279776Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:51:18.227071305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:51:18.233702985Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:51:19.24385322Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T11:51:19.528718381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:51:19.534006189Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:51:21.319261721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:51:21.32336068Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:51:22.468909056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:51:23.597875977Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T11:51:27.469154641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:51:27.469183974Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:51:27.469200692Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:51:27.469224004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:27.813990084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:51:27.81979008Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:51:27.953691006Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T11:51:28.665864786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:51:28.669421211Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T11:51:29.146034909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T11:51:31.425377865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:51:31.428976462Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T11:51:32.307742232Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T11:51:32.469118181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:33.345657116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:51:33.349379288Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:51:33.481334973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:51:33.486590589Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:51:33.632141418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:51:33.633988671Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:51:35.580598145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:51:35.584233346Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:51:36.662432516Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-02-16T11:51:37.234170653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:51:37.23819124Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T11:51:37.468350547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:38.388737241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:51:38.39257252Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:51:39.573861272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:51:39.577366895Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:51:39.703079733Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:51:40.261750361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:51:40.265220327Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:51:41.017437934Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-02-16T11:51:41.872003263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:51:41.875904679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T11:51:42.46861746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:51:45.372908993Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-02-16T11:51:46.226380018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:51:46.2299618Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:51:47.468597603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:51:49.726981197Z caller=checkpoint.go:378 msg="writing series" size="9.7 kB"
level=warn ts=2026-02-16T11:51:52.469157708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:51:54.017119812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:51:54.021981001Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T11:51:54.082568777Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=debug ts=2026-02-16T11:51:54.082592092Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=14
level=debug ts=2026-02-16T11:51:57.452668778Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:51:57.453736984Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:51:57.469061345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:51:57.469088396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:51:57.469096448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:51:57.469118273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:51:58.436575001Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-02-16T11:51:59.149412341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:52:00.214672232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33700"
level=info ts=2026-02-16T11:52:01.405354841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:52:01.409255631Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:52:02.216662883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:52:02.220189613Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:52:02.469301318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:02.793568837Z caller=checkpoint.go:378 msg="writing series" size="2.7 MB"
level=debug ts=2026-02-16T11:52:02.793592257Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=2
level=info ts=2026-02-16T11:52:02.860593854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:52:02.864168773Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:52:02.864191132Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:52:02.864214251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:52:02.867665846Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:52:04.07406118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:52:04.077557632Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:52:05.481926326Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:52:05.645525199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.5 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:52:05.650208205Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:52:07.073268773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:52:07.076611126Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:52:07.147079543Z caller=checkpoint.go:378 msg="writing series" size="374 kB"
level=warn ts=2026-02-16T11:52:07.468585443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:11.50130755Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=warn ts=2026-02-16T11:52:12.468581655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:52:13.062085855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:52:13.065961662Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:52:13.5868648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:52:13.590060049Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:52:13.590083428Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:52:13.590103283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:52:13.592936732Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:52:14.571659406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:52:14.575396081Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:52:15.315692932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:52:15.322248353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:52:15.322271562Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:52:15.322294404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:52:15.328030964Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:52:15.706499238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:52:15.70970011Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T11:52:15.856648813Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=warn ts=2026-02-16T11:52:17.469214854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:52:18.461340719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:52:18.464508215Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:52:18.672195852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:52:18.675493487Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T11:52:20.211570077Z caller=checkpoint.go:378 msg="writing series" size="535 kB"
level=debug ts=2026-02-16T11:52:20.211594782Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-02-16T11:52:20.322759858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:52:20.326057036Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:52:20.472827203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:52:20.47636531Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:52:20.500136579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:52:20.573309907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:52:20.577325418Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T11:52:22.468897506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:24.566044981Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T11:52:26.382228186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:52:26.386923071Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:52:26.514466788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:52:26.517697932Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:52:26.996106272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:52:26.999524082Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:52:27.468670708Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:52:27.468685341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:27.46869953Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:52:27.468708586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:52:28.1785522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:52:28.186823921Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:52:28.921159444Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T11:52:29.152300734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:52:29.216917894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:52:29.224114065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:52:30.173093844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:52:30.177103512Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:52:32.041020765Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:52:32.468836821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:33.276178711Z caller=checkpoint.go:378 msg="writing series" size="405 kB"
level=warn ts=2026-02-16T11:52:37.468564758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:37.631086253Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T11:52:38.330638732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:52:38.33468519Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:52:39.909094989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:52:39.91256013Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T11:52:41.985442232Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=warn ts=2026-02-16T11:52:42.469183179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:52:44.213402943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:52:44.216974871Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T11:52:45.452602652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:54400"
ts=2026-02-16T11:52:45.852701079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34252"
level=info ts=2026-02-16T11:52:45.940423945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:52:45.944916814Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:52:46.340226613Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=warn ts=2026-02-16T11:52:47.468975322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:52:47.994946733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:52:47.998684603Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T11:52:50.698394445Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-02-16T11:52:50.69841871Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.0 MB" series=7
level=info ts=2026-02-16T11:52:50.939161935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:52:50.94675187Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:52:52.468614899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:52:55.049842907Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=info ts=2026-02-16T11:52:56.751722474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:52:56.756381399Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:52:56.756408722Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:52:56.756432177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:52:56.759256204Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:52:57.449651902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:52:57.452342072Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:52:57.452915573Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=error ts=2026-02-16T11:52:57.453418853Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:52:57.468628134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:52:57.468631572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:52:57.468655682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:52:57.468663606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T11:52:58.33840713Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T11:52:59.155494253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:52:59.404766621Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-02-16T11:52:59.779096843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:52:59.784109574Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T11:53:00.219366292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50590"
level=info ts=2026-02-16T11:53:00.262229819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:53:00.265582061Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:53:01.079149669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:53:01.082667044Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T11:53:02.468721771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:53:03.759216514Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T11:53:04.210052939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.5 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:53:04.213563064Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:53:05.168203776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:53:05.4510222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:53:05.454317235Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:53:05.947791395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:53:05.951067653Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:53:06.80836151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:53:06.811511605Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:53:07.468988327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:53:08.114859732Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:53:10.220515618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:53:10.223413364Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:53:11.304246367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:53:11.307474399Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:53:11.525236199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:53:11.528609235Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:53:12.468705863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:53:12.469503573Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-02-16T11:53:12.469520834Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=5
level=info ts=2026-02-16T11:53:13.005502756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:53:13.008690788Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T11:53:14.771365803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:53:14.774894027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:53:16.823871348Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-02-16T11:53:17.469239478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:53:18.140849108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:53:18.144735491Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:53:19.067356064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:53:19.071091485Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:53:19.279685669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:53:19.282972428Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:53:20.091287373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:53:20.094690122Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:53:21.179168153Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=warn ts=2026-02-16T11:53:22.468573196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:53:25.129969486Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:53:25.534313957Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T11:53:27.468721153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:53:27.468752715Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:53:27.468761433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:53:27.468779831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:53:29.158507414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:53:29.219852277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:53:29.22449962Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:53:29.888542095Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-02-16T11:53:29.950857982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:53:29.956936472Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:53:32.468824346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:53:34.243623252Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=info ts=2026-02-16T11:53:34.716863255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:53:34.720885072Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T11:53:37.468425598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:53:38.369231786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:53:38.373043484Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T11:53:38.598861501Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-02-16T11:53:39.896219708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:53:39.899759064Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T11:53:42.468334352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:53:42.953292142Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-02-16T11:53:45.439948008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:53:45.444297692Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T11:53:45.858028319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42284"
level=info ts=2026-02-16T11:53:45.991367082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:53:45.995103021Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:53:47.308130527Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T11:53:47.468433496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:53:50.369988816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:53:50.376991118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:53:51.616583029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:53:51.620082531Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T11:53:51.662493446Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T11:53:52.468839553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:53:53.763590176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.5 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:53:53.767239871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T11:53:54.237387502Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:53:55.271596577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:53:55.274953048Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:53:56.01768287Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-02-16T11:53:56.065862253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:53:56.0693733Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T11:53:57.452354864Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:53:57.453409461Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:53:57.468679248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:53:57.468700253Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:53:57.468710417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:53:57.46872197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:53:59.161210263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:54:00.224576955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46816"
level=debug ts=2026-02-16T11:54:00.372708891Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=info ts=2026-02-16T11:54:00.802861397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:54:00.809672939Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T11:54:01.371026154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:54:01.374556422Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T11:54:02.469285816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:54:03.995661276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:54:03.99900785Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:54:03.999029261Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:54:03.999056084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:54:04.001785802Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T11:54:04.720459901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:54:04.727563421Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T11:54:05.216839894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:54:05.220087189Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:54:06.40809664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:54:06.411722275Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:54:07.468313398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:09.081868349Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T11:54:10.167082277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:54:10.170600579Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:54:11.183736077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:54:11.187100272Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:54:12.469254568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:13.437294143Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:54:15.644641028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:54:15.648770246Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:54:15.703849832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:54:15.707228511Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:54:17.291947184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.5 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:54:17.295884996Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T11:54:17.468457339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:17.792040685Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:54:18.545933701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:54:18.549258288Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:54:19.636534923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:54:19.6399201Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:54:20.229677832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:54:20.232963375Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T11:54:22.146182368Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T11:54:22.469379751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:23.070571792Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:54:26.501133936Z caller=checkpoint.go:378 msg="writing series" size="314 kB"
level=debug ts=2026-02-16T11:54:26.501157967Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=17
level=info ts=2026-02-16T11:54:27.468559236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:54:27.468590543Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:54:27.468603272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:54:27.468598307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:54:28.566317304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:54:28.569877032Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:54:29.126232628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:54:29.129752775Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T11:54:29.16383894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T11:54:30.623548623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:54:30.627661068Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:54:30.856498858Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T11:54:32.468852463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:54:32.622212741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="467 B" avg_comp="467 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:54:32.627180568Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:54:32.695094907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:54:32.698442444Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:54:35.211691881Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-02-16T11:54:36.213659918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:54:36.220400856Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:54:37.468518235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:39.568778026Z caller=checkpoint.go:378 msg="writing series" size="5.4 MB"
level=debug ts=2026-02-16T11:54:39.568814887Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.6 MB" series=3
level=info ts=2026-02-16T11:54:40.91962158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:54:40.924259269Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T11:54:41.978905039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T11:54:42.469243048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:43.920526665Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
ts=2026-02-16T11:54:45.459599123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59714"
level=info ts=2026-02-16T11:54:45.921498989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.5 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:54:45.924965154Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T11:54:47.46886797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:54:47.935614256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:54:47.939103232Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T11:54:48.276232277Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T11:54:48.48205982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:54:48.485276661Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:54:48.872362638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.6 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:54:48.875685786Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:54:50.639581925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:54:50.642631412Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:54:50.662966184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.5 kB" total_uncomp="56 kB" avg_uncomp="28 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:54:50.665800925Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T11:54:51.91691142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:54:51.920062472Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T11:54:52.469022773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:54:52.630350791Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=debug ts=2026-02-16T11:54:52.995119395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:54:56.984925617Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T11:54:57.035029153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:54:57.039793051Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:54:57.452669314Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:54:57.453732189Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:54:57.469060053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:54:57.469077301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:54:57.469084614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:54:57.469095835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:54:58.076350478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:54:58.079977082Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T11:54:59.167133091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:55:00.229333164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36644"
level=debug ts=2026-02-16T11:55:01.339934319Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=info ts=2026-02-16T11:55:01.482597661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:55:01.485895118Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T11:55:02.468723815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:55:02.990260898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:55:02.994494337Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:55:03.06287056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:55:03.066285369Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:55:03.372105055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:55:03.375337313Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:55:04.182737579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:55:04.186209173Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T11:55:05.6953283Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-02-16T11:55:07.46905003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:10.049628606Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=info ts=2026-02-16T11:55:10.999147211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:55:11.00279018Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T11:55:11.186261122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:55:11.189571128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T11:55:12.469151976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:14.405253453Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T11:55:16.405276001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:55:16.409027038Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:55:17.170235281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:55:17.177395785Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T11:55:17.468783518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:18.75993252Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-02-16T11:55:19.526448129Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:55:20.255638857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:55:20.259349459Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:55:22.425073945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="449 B" avg_comp="449 B" total_uncomp="350 B" avg_uncomp="350 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:55:22.428566029Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T11:55:22.468998082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:23.114273874Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T11:55:24.411099834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:55:24.414750942Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:55:27.468382198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:55:27.468387466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:27.468410466Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:55:27.468419435Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T11:55:27.469483618Z caller=checkpoint.go:390 msg="flushing series" totalSize="776 kB" series=10
level=info ts=2026-02-16T11:55:27.473104131Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-02-16T11:55:27.475473603Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006780806s
ts=2026-02-16T11:55:29.169798271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=warn ts=2026-02-16T11:55:32.468749688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:35.906780724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:55:35.908680084Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:55:36.566853685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:55:36.571327641Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T11:55:37.469319648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:55:39.542081517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:55:39.545376261Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:55:39.92531153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:55:39.928615247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:55:40.059868364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:55:40.063170006Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:55:40.279836971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:55:40.283168978Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:55:41.866646625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:55:41.869783539Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T11:55:42.025059988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:55:42.028237864Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T11:55:42.393270817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:55:42.398103618Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T11:55:42.469216068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:55:44.211327068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:55:44.214583024Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:55:44.253012489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:55:44.257262961Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:55:46.961297158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.7 kB" total_uncomp="57 kB" avg_uncomp="19 kB" idle=1 full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:55:46.966092742Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:55:47.468498402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:55:51.284017179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:55:51.287727801Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:55:52.398707682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:55:52.402149039Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:55:52.468492263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:55:54.79800507Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:55:56.869651987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:55:56.873671484Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:55:57.452654189Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:55:57.453707013Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:55:57.469100404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:55:57.469121084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:55:57.469115826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:55:57.46913074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:55:57.469129846Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T11:55:57.469258873Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-02-16T11:55:57.469425033Z caller=checkpoint.go:378 msg="writing series" size="518 B"
level=info ts=2026-02-16T11:55:58.732017129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:55:58.735397117Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T11:55:59.172844563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:56:01.82434809Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T11:56:01.871764929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:56:01.874929831Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:56:02.469052184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:03.129467106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:56:03.136080155Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:56:06.179572224Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-02-16T11:56:07.469197308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:08.636378188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:56:08.639768029Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:56:08.639787401Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:08.639828559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:56:08.642615637Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T11:56:09.688427926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:56:09.691875889Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:56:10.534333461Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-02-16T11:56:12.469308719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:13.490580055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" idle=1 full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:56:13.494498509Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:56:13.744097297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:56:13.747551161Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:56:14.889761727Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=info ts=2026-02-16T11:56:15.776261984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:56:15.779786013Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T11:56:17.469240896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:56:19.078077925Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:56:19.244495603Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T11:56:21.66409838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:56:21.667280597Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T11:56:22.468866203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:56:22.667334978Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:56:23.598705072Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T11:56:25.309000867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:56:25.312484296Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:56:25.312509234Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:25.312535757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:56:25.315295803Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:56:25.440594712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:56:25.444061598Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c6638380a%3A19c6638380b%3A3b9c21c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T11:56:25.44408244Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:25.613584714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="465 B" avg_comp="465 B" total_uncomp="222 kB" avg_uncomp="222 kB" idle=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T11:56:25.616945196Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T11:56:27.099088529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:56:27.102713893Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T11:56:27.102745946Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:27.10277676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:56:27.105925987Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:56:27.468523573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:56:27.468534623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:56:27.468552816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:56:27.468561804Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T11:56:27.953787083Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T11:56:29.175948061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:56:30.148214239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:56:30.151318344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T11:56:30.151342306Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:30.15136474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:56:30.154126171Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:56:32.309103265Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T11:56:32.468744255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:32.511855911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:56:32.515323281Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:56:32.515344714Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:32.515367139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:56:32.518164162Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:56:34.497761255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:56:34.501242229Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:56:34.501261413Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:34.501281296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:56:34.504648288Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:56:35.37096888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:56:35.375368045Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:56:35.375388847Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:35.37541204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:56:35.378225312Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T11:56:36.59559254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:56:36.5988953Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:56:36.663355532Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=info ts=2026-02-16T11:56:37.24060603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:56:37.243876586Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:56:37.298062527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:56:37.302013616Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T11:56:37.468697121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:38.210835898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:56:38.214271326Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:56:38.214290326Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:38.214312168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:56:38.21700958Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:56:38.851211572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:56:38.854404044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T11:56:38.854424738Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:38.854446003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:56:38.857122882Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:56:39.39399937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:56:39.397356222Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:56:39.397375783Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:39.39739832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:56:39.400155466Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:56:40.192459969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:56:40.195766926Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:56:41.018201127Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=warn ts=2026-02-16T11:56:42.468943319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:43.812550592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:56:43.815941854Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:56:44.064843009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:56:44.067911749Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:56:44.699851384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T11:56:44.703023349Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T11:56:45.372902738Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-02-16T11:56:45.592847682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:56:45.59594273Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T11:56:45.871120368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40862"
level=info ts=2026-02-16T11:56:46.050843147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.7 kB" total_uncomp="57 kB" avg_uncomp="19 kB" full=3 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:56:46.053947927Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T11:56:47.468334713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:56:49.064984911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:56:49.069741212Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:56:49.340842052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:56:49.345009706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-02-16T11:56:49.346796385Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:56:49.727943733Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T11:56:49.731056362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:56:49.734264876Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:56:49.73428306Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:49.734304287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:56:49.736996184Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:56:50.958208482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:56:50.96154874Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T11:56:52.07486273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:56:52.078332216Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:56:52.078359558Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:56:52.078388703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:56:52.081254865Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:56:52.424105978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.4 kB" avg_comp="9.4 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:56:52.427350314Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T11:56:52.46871838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:56:54.082750012Z caller=checkpoint.go:378 msg="writing series" size="267 kB"
level=debug ts=2026-02-16T11:56:54.082773481Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=14
level=info ts=2026-02-16T11:56:54.318224043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:56:54.321481928Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T11:56:55.474757767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:56:55.478023668Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:56:55.713798011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:56:55.717203987Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:56:57.452204376Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:56:57.453251911Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:56:57.468588166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:56:57.468613019Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:56:57.468624361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:56:57.46863027Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:56:58.43811983Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T11:56:58.722208199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:56:58.725322293Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T11:56:59.178557806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:57:00.237943986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52146"
level=info ts=2026-02-16T11:57:00.636292127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:57:00.639479497Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T11:57:02.102444551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T11:57:02.4692126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:57:02.795098521Z caller=checkpoint.go:378 msg="writing series" size="4.6 MB"
level=debug ts=2026-02-16T11:57:02.795121838Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.6 MB" series=2
level=info ts=2026-02-16T11:57:03.948695056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:57:03.952246719Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:57:04.377326462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:57:04.380655375Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:57:05.864820585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:57:05.871176247Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:57:06.274261104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:57:06.277714441Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:57:06.529489623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T11:57:06.532838791Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T11:57:07.1014253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:57:07.104748442Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:57:07.147022634Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=warn ts=2026-02-16T11:57:07.469125874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:08.198387141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:57:08.201614739Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:57:11.390856947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:57:11.394376261Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T11:57:11.501867866Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=info ts=2026-02-16T11:57:11.657099301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:57:11.660442611Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T11:57:12.468618375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:12.722023682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:57:12.725063591Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T11:57:15.469455868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36924"
level=debug ts=2026-02-16T11:57:15.857294164Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=warn ts=2026-02-16T11:57:17.468606509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:19.844320127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:57:19.848474326Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T11:57:20.21247471Z caller=checkpoint.go:378 msg="writing series" size="521 kB"
level=info ts=2026-02-16T11:57:20.920663777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.5 kB" avg_comp="9.5 kB" total_uncomp="45 kB" avg_uncomp="45 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:57:20.924147859Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T11:57:22.101172906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T11:57:22.19731319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:57:22.201192072Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T11:57:22.46927772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:23.882425351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:57:23.886187086Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:57:24.567106361Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T11:57:26.542838928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:57:26.546275301Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:57:27.468776854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:57:27.468826339Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:57:27.468835485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:57:27.468778738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:28.628633021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:57:28.632109788Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T11:57:28.922028443Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T11:57:29.18091099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:57:29.447089657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:57:29.450521696Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:57:29.450543332Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:57:29.450562879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:57:29.45343375Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T11:57:30.240708119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50646"
level=warn ts=2026-02-16T11:57:32.468535548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:32.844738236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:57:32.84831426Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:57:33.276947476Z caller=checkpoint.go:378 msg="writing series" size="603 kB"
level=debug ts=2026-02-16T11:57:33.276972211Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-02-16T11:57:36.555015376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:57:36.56219241Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:57:36.617331444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=3 total_comp="11 kB" avg_comp="3.7 kB" total_uncomp="57 kB" avg_uncomp="19 kB" full=3 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:57:36.620493581Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c6637d559%3A19c66386604%3A3c1a0614?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T11:57:36.620513485Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:57:36.620536623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="316 kB" avg_comp="316 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" idle=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T11:57:36.627531979Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T11:57:37.246874565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="115 kB" avg_uncomp="115 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:57:37.25068595Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T11:57:37.468600658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:57:37.63137914Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T11:57:41.77919343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:57:41.782448974Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T11:57:41.986116125Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=warn ts=2026-02-16T11:57:42.468936042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:43.54797706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:57:43.554269767Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:57:45.475732915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:57:45.47935633Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T11:57:45.876482088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33786"
level=info ts=2026-02-16T11:57:46.03984403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:57:46.042837206Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:57:46.042855372Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:57:46.042874994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:57:46.045639102Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:57:46.341242169Z caller=checkpoint.go:378 msg="writing series" size="7.3 kB"
level=info ts=2026-02-16T11:57:46.827818593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:57:46.831056307Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T11:57:47.469325005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:57:47.560785575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:57:50.698287791Z caller=checkpoint.go:378 msg="writing series" size="6.3 MB"
level=debug ts=2026-02-16T11:57:50.698311002Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.4 MB" series=4
level=debug ts=2026-02-16T11:57:51.395656883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:57:52.469034343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:52.631329371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:57:52.636131597Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:57:52.707911954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:57:52.710888151Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T11:57:52.801226116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:57:52.804387309Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T11:57:55.050439689Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=info ts=2026-02-16T11:57:55.858930176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T11:57:55.862852417Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T11:57:57.452872419Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:57:57.452875366Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:57:57.468261856Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:57:57.468276422Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:57:57.46828397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:57:57.468323333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:57:58.604445847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:57:58.607819436Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T11:57:59.18345904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:57:59.405665927Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
ts=2026-02-16T11:58:00.243238721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44908"
level=info ts=2026-02-16T11:58:01.008156145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:58:01.011569099Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:58:02.173931432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:58:02.178083111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:58:02.468778922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:58:03.224438834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T11:58:03.227986522Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T11:58:03.760164185Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T11:58:03.789285206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:58:03.792773557Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T11:58:04.007359187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:58:04.011078497Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:58:04.011101855Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:58:04.011125679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:58:04.01421788Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T11:58:07.164601914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" idle=1 full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:58:07.170188173Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T11:58:07.468479461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:58:08.115695682Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T11:58:08.677991296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:58:08.681455701Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:58:11.636278273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:58:11.640221193Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T11:58:12.468702648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:58:12.470535941Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-02-16T11:58:12.47055248Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=5
level=info ts=2026-02-16T11:58:15.852290332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:58:15.855824203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T11:58:16.824781163Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-02-16T11:58:17.468334687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:58:18.813149203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:58:18.816486608Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:58:18.957017171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:58:18.960351194Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T11:58:20.815628622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:58:20.81913837Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T11:58:20.959783578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:58:21.179580341Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=info ts=2026-02-16T11:58:22.28502214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:58:22.289132415Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T11:58:22.468928234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:58:24.299135092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T11:58:24.304393898Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T11:58:24.606334762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:58:24.609951997Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T11:58:25.534676898Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T11:58:25.879917428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:58:25.883076455Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T11:58:27.468553692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T11:58:27.468564423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:58:27.468581885Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:58:27.468591152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:58:27.921241361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:58:27.924465623Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T11:58:29.186682622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:58:29.889322698Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=warn ts=2026-02-16T11:58:32.4683778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:58:33.063666577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:58:33.06735195Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T11:58:34.244650632Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-02-16T11:58:36.502277541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:58:36.5062925Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T11:58:37.468532849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:58:38.598726793Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-02-16T11:58:39.909238676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:58:39.913860975Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:58:39.913880583Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:58:39.913902552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:58:39.916959464Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:58:40.05852695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:58:40.062790122Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T11:58:42.07623214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:58:42.083157947Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:58:42.083188389Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:58:42.083225195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:58:42.089594515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:58:42.437841142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:58:42.441279606Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:58:42.441300959Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:58:42.441322652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:58:42.444172499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T11:58:42.468495287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:58:42.954109326Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-02-16T11:58:43.545064033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T11:58:44.534053021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:58:44.538101014Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T11:58:47.262738373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:58:47.266579681Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:58:47.30868743Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T11:58:47.469130634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:58:48.368525611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:58:48.372093266Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T11:58:48.928863634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:58:48.932129915Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:58:51.138241443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:58:51.141640512Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T11:58:51.663415755Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-02-16T11:58:51.724627619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:58:52.468543275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:58:52.93905142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:58:52.945672262Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T11:58:54.850270025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:58:54.855037905Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T11:58:56.018123174Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=info ts=2026-02-16T11:58:56.468861829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:58:56.472826883Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T11:58:57.452571139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T11:58:57.453627007Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T11:58:57.469030204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:58:57.469049471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:58:57.469056778Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T11:58:57.469069078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T11:58:59.189256762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T11:59:00.373202143Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=info ts=2026-02-16T11:59:00.579785193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:59:00.583458423Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:59:00.739886994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T11:59:00.743793746Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T11:59:01.838008596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:59:01.841331499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T11:59:02.469012455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:02.632036241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:59:02.635371914Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:59:04.727923438Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-02-16T11:59:05.544236326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T11:59:05.547848877Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T11:59:07.468852779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:08.63682376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T11:59:08.64018581Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T11:59:08.924827878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:59:08.928225159Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T11:59:09.082848938Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T11:59:10.986145144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T11:59:10.990895887Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T11:59:12.469165152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:59:13.437698571Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-02-16T11:59:15.883980918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60796"
level=info ts=2026-02-16T11:59:16.520019099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:59:16.523595809Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T11:59:16.614127808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T11:59:16.617388059Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T11:59:16.779851391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:59:16.783077556Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:59:16.783096586Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T11:59:16.783117221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:59:16.785884299Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T11:59:17.469055129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:59:17.792840251Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-02-16T11:59:18.763311526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:59:18.765226854Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:59:21.0381655Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T11:59:22.147251039Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-02-16T11:59:22.46842651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:22.588782144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T11:59:22.592548313Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T11:59:23.134253281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T11:59:23.138389839Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T11:59:24.225414414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T11:59:24.228740876Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T11:59:26.482855145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.5 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T11:59:26.486602654Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T11:59:26.50287612Z caller=checkpoint.go:378 msg="writing series" size="423 kB"
level=debug ts=2026-02-16T11:59:26.502893472Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=17
level=info ts=2026-02-16T11:59:27.468580835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:59:27.468609518Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T11:59:27.468594429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:27.468618885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T11:59:29.192748268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:59:29.55555801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T11:59:29.562453304Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T11:59:30.857028236Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-02-16T11:59:32.468594234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:34.486450182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T11:59:34.490941342Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T11:59:35.211639735Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=info ts=2026-02-16T11:59:35.315041222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T11:59:35.319038749Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T11:59:36.172493311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T11:59:36.18409388Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T11:59:37.34153126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T11:59:37.345453359Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T11:59:37.468367758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:38.521404617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T11:59:38.524727229Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T11:59:39.574007462Z caller=checkpoint.go:378 msg="writing series" size="9.8 MB"
level=debug ts=2026-02-16T11:59:39.57403224Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=3
level=info ts=2026-02-16T11:59:40.104530526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T11:59:40.107943435Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T11:59:41.355645634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T11:59:41.360192177Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T11:59:42.468640866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:59:43.921665251Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-02-16T11:59:43.946940348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T11:59:43.951185191Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T11:59:45.477326746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52146"
ts=2026-02-16T11:59:45.887389463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44428"
level=warn ts=2026-02-16T11:59:47.469203346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T11:59:48.276759898Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=debug ts=2026-02-16T11:59:48.715658093Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T11:59:52.469296474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:52.628642104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T11:59:52.630885103Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=error ts=2026-02-16T11:59:52.633439586Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T11:59:53.050154499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.6 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T11:59:53.055077469Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T11:59:55.014712081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T11:59:55.018012549Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T11:59:56.453244363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T11:59:56.458231516Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T11:59:56.986130432Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T11:59:57.452907468Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T11:59:57.457578964Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=warn ts=2026-02-16T11:59:57.469021091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T11:59:57.469025515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T11:59:57.469065931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T11:59:57.469077854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T11:59:58.479230941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T11:59:58.482671672Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T11:59:59.196035473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T11:59:59.373034417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T11:59:59.377996374Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:00:00.845906047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:00:00.857062072Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:00:01.117296581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:00:01.120742133Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:00:01.345178476Z caller=checkpoint.go:378 msg="writing series" size="487 kB"
level=warn ts=2026-02-16T12:00:02.469312146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:03.470941455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:00:03.47645457Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:00:05.084648626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:00:05.088923767Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T12:00:05.696152884Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=warn ts=2026-02-16T12:00:07.469299919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:00:10.054114385Z caller=checkpoint.go:378 msg="writing series" size="268 kB"
level=warn ts=2026-02-16T12:00:12.469277223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:14.379441204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:00:14.383749357Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:00:14.405859039Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:00:14.710042122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:00:14.713472973Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:00:16.259839353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T12:00:17.468409722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:17.755949911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:00:17.762346611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:00:18.760757795Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T12:00:18.960602066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="575 B" avg_comp="287 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:00:18.964530827Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:00:19.911856898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:00:19.915253918Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:00:20.329674594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:00:20.334739383Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T12:00:22.033267051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:00:22.037865937Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:00:22.468929206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:00:23.11555316Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-02-16T12:00:24.412134468Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:00:26.681252377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:00:26.686142715Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T12:00:27.468410327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:27.468431518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:00:27.468463055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:00:27.468470828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T12:00:27.469526945Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=10
level=info ts=2026-02-16T12:00:27.695778966Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-02-16T12:00:27.699185409Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.229792675s
ts=2026-02-16T12:00:29.199045647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:00:29.367863417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:00:29.37182687Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T12:00:30.254966092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42266"
level=warn ts=2026-02-16T12:00:32.468753104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:33.589734574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:00:33.594011755Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:00:34.466267638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:00:34.470762699Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:00:37.469237902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:38.603393309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:00:38.6076345Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:00:42.46908909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:43.5200377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:00:43.523748974Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:00:44.379744281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:00:44.384641654Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T12:00:45.481560174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55532"
level=info ts=2026-02-16T12:00:47.195620857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:00:47.200827485Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T12:00:47.469198496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:50.06449744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:00:50.068916156Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:00:50.068943458Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:00:50.068976655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:00:50.072340831Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:00:52.13443417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:00:52.138986795Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:00:52.248664597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:00:52.253504685Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T12:00:52.468757985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:53.794121088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:00:53.797339588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:00:55.997746257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:00:56.00105545Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:00:57.452850986Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:00:57.452858215Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T12:00:57.454588859Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T12:00:57.454647928Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968047 n=1
level=debug ts=2026-02-16T12:00:57.45470144Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968047 n=1
level=warn ts=2026-02-16T12:00:57.468731785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:00:57.468792599Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T12:00:57.468852458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:00:57.468868789Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:00:57.468884492Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-02-16T12:00:57.468876882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T12:00:57.469030984Z caller=checkpoint.go:378 msg="writing series" size="518 B"
level=debug ts=2026-02-16T12:00:57.767204078Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:00:57.80244907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:00:57.80707689Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:00:58.546041265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:00:58.550117269Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T12:00:59.201969848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:01:00.221756759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:01:00.224992261Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:01:01.824875924Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T12:01:02.46835337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:02.722703674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:01:02.726934258Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:01:06.179770952Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-02-16T12:01:06.90171746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:01:06.905058352Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:01:07.029334831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:01:07.03348152Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T12:01:07.468630671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:08.106386698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:01:08.112553447Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:01:08.35737175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:01:08.360779495Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:01:09.800358337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:01:09.803577844Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:01:09.833651826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:01:09.837088993Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:01:09.88471636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:01:10.01092378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:01:10.014373261Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:01:10.534070094Z caller=checkpoint.go:378 msg="writing series" size="281 kB"
level=warn ts=2026-02-16T12:01:12.46874468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:01:14.889388145Z caller=checkpoint.go:378 msg="writing series" size="567 kB"
ts=2026-02-16T12:01:15.895940336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60630"
level=info ts=2026-02-16T12:01:17.268863341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:01:17.274530988Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:01:17.451653427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:01:17.455621331Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:01:17.468707217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:01:19.244169882Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=warn ts=2026-02-16T12:01:22.46922772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:01:23.599051195Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T12:01:23.958869874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:01:23.962872576Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:01:27.469128255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:01:27.469159414Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T12:01:27.469147822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:27.469170207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T12:01:27.953089127Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T12:01:29.204421553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:01:30.204260688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:01:30.208069593Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:01:31.260873699Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:01:32.118856329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:01:32.122521015Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T12:01:32.308079153Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T12:01:32.469399433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:36.193422153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:01:36.197467775Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T12:01:36.662968147Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=debug ts=2026-02-16T12:01:36.66301641Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-02-16T12:01:37.468993593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:38.704113876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:01:38.707541414Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:01:40.859478379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:01:40.863046978Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:01:41.017834026Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T12:01:41.242951198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:01:41.247548711Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T12:01:42.468820199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:42.759497032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:01:42.763307613Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:01:42.897032941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:01:42.900688771Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:01:44.614366068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:01:44.618050413Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T12:01:45.372310934Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-02-16T12:01:47.3648703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T12:01:47.46901032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:48.764614681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:01:48.767981893Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:01:49.184990143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:01:49.188495592Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:01:49.727570847Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T12:01:50.844262794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:01:50.848465756Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:01:52.211046235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:01:52.214713464Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T12:01:52.468726083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:01:54.083210169Z caller=checkpoint.go:378 msg="writing series" size="693 kB"
level=info ts=2026-02-16T12:01:54.681696552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:01:54.685350452Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:01:56.892076756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:01:56.896539399Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:01:57.453134766Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:01:57.453132662Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=warn ts=2026-02-16T12:01:57.468644616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:01:57.468638305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:01:57.468671171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:01:57.468682404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:01:57.986080869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:01:57.989968863Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:01:58.011072695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:01:58.018649603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:01:58.437073725Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T12:01:58.582833822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:01:58.59283407Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T12:01:59.207659021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:02:00.036299516Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:02:00.838264454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:02:00.84279262Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T12:02:02.46902909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:02.795627604Z caller=checkpoint.go:378 msg="writing series" size="7.0 MB"
level=debug ts=2026-02-16T12:02:02.795666371Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.8 MB" series=6
level=info ts=2026-02-16T12:02:03.934088187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:02:03.938618892Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:02:07.147150904Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=warn ts=2026-02-16T12:02:07.468646448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:11.501858469Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T12:02:11.597098677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:02:11.601460599Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:02:12.468660158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:02:14.729690673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:02:14.734584234Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T12:02:15.487444764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45362"
level=debug ts=2026-02-16T12:02:15.856915466Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
ts=2026-02-16T12:02:15.900990379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54688"
level=info ts=2026-02-16T12:02:17.051472807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:02:17.056172795Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:02:17.46910297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:20.211435279Z caller=checkpoint.go:378 msg="writing series" size="744 kB"
level=debug ts=2026-02-16T12:02:20.211472725Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-02-16T12:02:22.469006138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:24.566778488Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:02:24.686268291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:02:24.689917022Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:02:24.903654853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:02:25.421297627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:02:25.424950879Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:02:25.935707195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:02:25.939901493Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:02:25.9399332Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:25.939978664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:02:25.943648824Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:02:27.468262284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:02:27.468302844Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:02:27.468313985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:02:27.468331637Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:02:27.504483436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:02:27.508483758Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T12:02:28.558143255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T12:02:28.921135604Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T12:02:29.211155877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:02:29.557941292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:02:29.562371416Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:02:29.932540199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:02:29.937345261Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:02:31.269351438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:02:31.272624947Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:02:31.272642928Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:31.272662485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:02:31.275485973Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:02:31.971857707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="4.8 kB" avg_uncomp="4.8 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:02:31.97534142Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:02:31.975360913Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:31.975393394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:02:31.978257071Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T12:02:32.469313778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:33.276022932Z caller=checkpoint.go:378 msg="writing series" size="566 kB"
level=info ts=2026-02-16T12:02:33.470708851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:02:33.474203572Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:02:37.469154428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:37.63053367Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T12:02:39.402604288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:02:39.406450935Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:02:40.577948864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:02:40.581440546Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:02:40.58145812Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:40.581478368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="54 kB" avg_uncomp="54 kB" idle=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:02:40.584306495Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T12:02:41.986228025Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=info ts=2026-02-16T12:02:42.328143319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:02:42.332108625Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T12:02:42.332134459Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:42.332164157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:02:42.335312094Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:02:42.469006226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:02:42.506164712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:02:42.509449045Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:02:43.627284875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:02:43.630426663Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:02:44.948695924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:02:44.95210337Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:02:45.013224079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:02:45.016656642Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:02:45.410933204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:02:45.414466321Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T12:02:45.489976625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49526"
level=info ts=2026-02-16T12:02:46.034933637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:02:46.039329605Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:02:46.340429765Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=warn ts=2026-02-16T12:02:47.469135278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:02:49.032026272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:02:49.038397622Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:02:49.677510376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:02:49.680731089Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:02:49.68075144Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:49.680773709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:02:49.684035238Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:02:50.141657312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:02:50.144925359Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:02:50.697758166Z caller=checkpoint.go:378 msg="writing series" size="6.7 MB"
level=debug ts=2026-02-16T12:02:50.697784603Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.4 MB" series=7
level=warn ts=2026-02-16T12:02:52.469126689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:02:54.233266522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:02:54.238729757Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:02:54.23876096Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:02:54.238796092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:02:54.241837535Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:02:55.050696975Z caller=checkpoint.go:378 msg="writing series" size="370 kB"
level=info ts=2026-02-16T12:02:55.683233239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:02:55.686446378Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T12:02:56.324405158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:02:56.564305596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:02:56.567567402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:02:57.452534448Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:02:57.453591244Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:02:57.469067875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:02:57.469086744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:02:57.46909823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:02:57.469108213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:02:58.956094565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:02:58.960543816Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T12:02:59.214120684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:02:59.405128523Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
ts=2026-02-16T12:03:00.263404595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39046"
level=info ts=2026-02-16T12:03:00.538557887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:03:00.543846461Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:03:00.729258828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:03:00.732821491Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:03:02.468992508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:03.760128235Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T12:03:04.692940122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:03:04.696364501Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T12:03:07.468368722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:07.61764238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:03:07.61935183Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:03:08.115113822Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=info ts=2026-02-16T12:03:09.481150875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:03:09.484492549Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T12:03:09.48451345Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:09.484540579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.7 MB" avg_comp="2.4 MB" total_uncomp="80 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:03:09.490778951Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:03:10.573011662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:03:10.576475899Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:03:10.576496978Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:10.576518412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:03:10.579449586Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T12:03:12.468900937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:12.470766156Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-02-16T12:03:12.470782217Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=5
level=info ts=2026-02-16T12:03:12.824233773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:03:12.827583342Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:03:15.193085941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:03:15.196550131Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T12:03:15.492872716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:42104"
level=info ts=2026-02-16T12:03:16.527861858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:03:16.531341595Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:03:16.531361137Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:16.531382568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:03:16.534347475Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T12:03:16.824174357Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T12:03:17.114146109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:03:17.117460518Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T12:03:17.468940776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:03:18.036514104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:03:18.040112236Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:03:19.38585265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:03:19.389115945Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:03:20.329751286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:03:20.333337854Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:03:20.333359301Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:20.333384848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:03:20.336187865Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:03:21.179615699Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-02-16T12:03:21.35571764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.7 MB" avg_comp="2.4 MB" total_uncomp="80 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:03:21.363030795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T12:03:22.46864332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:03:24.577293272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:03:24.581079824Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:03:24.581099514Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:24.581120633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:03:24.584816502Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:03:25.533879932Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T12:03:25.843357571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:03:25.847054316Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:03:25.847075086Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:25.84709766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:03:25.849931642Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:03:26.020921272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:03:26.024401661Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:03:26.601115841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:03:26.604375141Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:03:27.035333721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:03:27.038538274Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:03:27.468372934Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:03:27.468379818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:27.468400347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:03:27.468408878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T12:03:28.49890043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T12:03:29.216536958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:03:29.889311137Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-02-16T12:03:30.266007362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49926"
level=info ts=2026-02-16T12:03:30.373739027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:03:30.377650205Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:03:32.469287418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:03:32.48542652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:03:32.489115507Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:03:34.243497385Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=info ts=2026-02-16T12:03:35.329694024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:03:35.333277104Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:03:36.849070985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:03:36.853619029Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:03:37.469265238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:38.598379581Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=info ts=2026-02-16T12:03:39.020345886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:03:39.023651186Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:03:39.117454647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" idle=1 full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:03:39.121201725Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:03:40.532493678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:03:40.536196102Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:03:40.605471197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:03:40.608840279Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:03:40.859403475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:03:40.862605744Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:03:40.862625881Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:03:41.163547013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:03:41.166644084Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:03:42.028335556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:03:42.032092447Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:03:42.468741982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:42.954019606Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=info ts=2026-02-16T12:03:44.119615587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:03:44.126022873Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T12:03:45.496026832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52376"
level=debug ts=2026-02-16T12:03:47.307965392Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T12:03:47.469206077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:03:47.48828678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:03:47.492429469Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:03:51.04876756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:03:51.052403674Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:03:51.663294588Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T12:03:52.468764891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:03:54.33891551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:03:54.342462632Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:03:54.728714992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.7 MB" avg_comp="2.4 MB" total_uncomp="80 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:03:54.758902251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:03:54.865308179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:03:54.868847417Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:03:55.009111603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:03:55.0130012Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:03:55.848473998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:03:55.852103012Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T12:03:56.017607325Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=info ts=2026-02-16T12:03:57.277006325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:03:57.280441431Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:03:57.452519961Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:03:57.453580773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:03:57.469053873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:03:57.469082429Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:03:57.469095834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:03:57.469114469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:03:58.647336106Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:03:58.647381671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="524 kB" avg_comp="524 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:03:58.651147655Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-02-16T12:03:59.221970597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:03:59.517305625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:00.268982227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47986"
level=debug ts=2026-02-16T12:04:00.372684482Z caller=checkpoint.go:378 msg="writing series" size="368 kB"
level=debug ts=2026-02-16T12:04:00.372707764Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-02-16T12:04:02.469077599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:04:02.878770018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="222 kB" avg_uncomp="222 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:04:02.883834844Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:04:04.161491669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:04:04.165097741Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:04:04.727916806Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=info ts=2026-02-16T12:04:05.141089393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:04:05.144502306Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:04:06.654139586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:04:06.657586315Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T12:04:07.468300205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:04:09.082760726Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T12:04:09.874056905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:04:09.878901416Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T12:04:12.468572661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:04:13.437359363Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-02-16T12:04:13.622817489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:04:13.626491766Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T12:04:15.498723389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:35174"
level=info ts=2026-02-16T12:04:15.883677802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:04:15.887462713Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T12:04:17.468800535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:04:17.792234231Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T12:04:18.906664735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:04:18.910679966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:04:20.12013168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:04:20.123602719Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:04:20.831697685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:04:20.835653353Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:04:22.041110829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:04:22.045458194Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:04:22.045481413Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:04:22.045506645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:04:22.04934065Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:04:22.146675937Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-02-16T12:04:22.468753832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:04:24.129239029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:04:24.132701849Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:04:26.50199414Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=info ts=2026-02-16T12:04:27.468834421Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:04:27.468845387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:04:27.468870896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:04:27.468881853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T12:04:29.224738718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:04:30.27198892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39352"
level=info ts=2026-02-16T12:04:30.672538184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:04:30.676769982Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:04:30.856505538Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=info ts=2026-02-16T12:04:32.444484389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:04:32.451776257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:04:32.468936153Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:04:33.779720537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:04:33.783646935Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:04:33.78367123Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:04:33.78370174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:04:33.786461578Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:04:33.928956241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:04:33.965119145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:04:33.968622067Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:04:33.968637594Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:04:34.08401256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:04:34.087252996Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:04:35.211397415Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T12:04:36.257219344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.7 MB" avg_comp="2.4 MB" total_uncomp="80 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:04:36.270145357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T12:04:37.469296147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:04:37.533405954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.6 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:04:37.536954424Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:04:38.277269074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:04:38.281564934Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:04:39.546551039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.3 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:04:39.549818083Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:04:39.574847272Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-02-16T12:04:39.574868227Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=9
level=info ts=2026-02-16T12:04:41.583963901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:04:41.586765993Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:04:41.637870499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:04:41.641526364Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T12:04:42.469276846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:04:43.554084647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:04:43.921323178Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-02-16T12:04:44.498298646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:04:44.502041133Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:04:44.869254703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:04:44.872727083Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:04:47.468764936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:04:47.58408965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:04:47.587654739Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:04:47.685855689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" idle=1 full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:04:47.689319587Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:04:48.276106958Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T12:04:49.539041312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:04:49.542347609Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:04:50.127928865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:04:50.13148966Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:04:52.468597671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:04:52.631095434Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T12:04:53.321633922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:04:53.32523577Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:04:54.46485555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:04:54.468796442Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:04:56.985997491Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=debug ts=2026-02-16T12:04:57.452769233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:04:57.452782301Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:04:57.468760111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:04:57.468793684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:04:57.468828404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:04:57.468834708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T12:04:59.228373115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:05:01.341051391Z caller=checkpoint.go:378 msg="writing series" size="487 kB"
level=warn ts=2026-02-16T12:05:02.468355286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:05:02.838288576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:05:02.843221655Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:05:02.859649401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:05:02.863174227Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:05:03.772103545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:05:03.776608198Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:05:05.023291357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:05:05.029149512Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T12:05:05.695750522Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-02-16T12:05:05.865493187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:05:05.869282312Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:05:06.8233051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:05:07.468513315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:05:08.104706324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:05:08.108255051Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:05:08.108275277Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:05:08.108296366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="593 B" avg_comp="296 B" total_uncomp="690 B" avg_uncomp="345 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:05:08.111312165Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:05:10.050479624Z caller=checkpoint.go:378 msg="writing series" size="398 kB"
level=debug ts=2026-02-16T12:05:10.050504892Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T12:05:10.625546663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:05:10.629533834Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:05:11.069857648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:05:11.073646252Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T12:05:12.469336676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:05:14.40488992Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T12:05:16.11283098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:05:16.116615543Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:05:16.23565261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:05:16.239088016Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T12:05:17.469224014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:05:18.759710076Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T12:05:21.300856163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:05:21.305773054Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:05:22.4689722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:05:23.114549408Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T12:05:25.024760634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:05:25.028923601Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:05:27.468979116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:05:27.469010242Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:05:27.46901869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:05:27.469013254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:05:27.469032558Z caller=checkpoint.go:390 msg="flushing series" totalSize="29 kB" series=3
level=info ts=2026-02-16T12:05:27.591344369Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-02-16T12:05:27.594525207Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.125528371s
level=info ts=2026-02-16T12:05:27.887493094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:05:27.891337397Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:05:28.085284509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="338 B" total_uncomp="1.0 kB" avg_uncomp="343 B" idle=1 full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:05:28.089031022Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:05:28.289902354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:05:28.297376611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:05:28.297399032Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:05:28.29742352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:05:28.303568556Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T12:05:29.231887458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:05:30.276948628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38210"
level=warn ts=2026-02-16T12:05:32.468930394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:05:33.071111792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="103 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:05:33.08276925Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T12:05:33.473345885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:05:34.183610556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:05:34.187096814Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:05:34.922763331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:05:34.926371731Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:05:36.035058118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:05:36.039533145Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:05:36.442513481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:05:36.446868479Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:05:37.270002573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="14 kB" avg_comp="7.2 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:05:37.273456045Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T12:05:37.468334059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:05:39.151127851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:05:39.155833946Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:05:42.468636298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:05:42.51693362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:05:43.621855903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:05:43.62526005Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:05:43.713726568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:05:43.717306678Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:05:43.717324643Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:05:43.717344662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:05:43.720157822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:05:45.254132335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:05:45.258371272Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:05:45.258394417Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:05:45.258420165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:05:45.261212745Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:05:45.366826788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:05:45.374156777Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:05:45.909659586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:05:45.913237435Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:05:47.46837448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T12:05:52.469220472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:05:55.334820424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:05:55.34040219Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:05:56.26979982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:05:56.273362741Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:05:57.452863657Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:05:57.453209551Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:05:57.468684823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T12:05:57.468701648Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-02-16T12:05:57.468696348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:05:57.468728158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:05:57.468742839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:05:57.468852904Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-02-16T12:05:57.469025088Z caller=checkpoint.go:378 msg="writing series" size="518 B"
level=debug ts=2026-02-16T12:05:57.778216107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T12:05:59.23451524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:05:59.984299791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:05:59.987693409Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T12:06:01.824310156Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T12:06:02.116560081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:06:02.120703214Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:06:02.187941378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:06:02.191831325Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:06:02.203845572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:06:02.207389329Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:06:02.468825549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:03.609325968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:06:03.61364358Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:06:03.81627436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:06:03.820184969Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:06:05.307182318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:06:05.310903117Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:06:06.178885876Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-02-16T12:06:07.256762402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:06:07.261469446Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:06:07.46858683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:09.692862364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:06:09.696914692Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:06:10.533902553Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=info ts=2026-02-16T12:06:12.011633169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:06:12.015074508Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T12:06:12.468618262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:06:14.889378021Z caller=checkpoint.go:378 msg="writing series" size="668 kB"
level=debug ts=2026-02-16T12:06:14.889407034Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
ts=2026-02-16T12:06:15.919311767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53796"
level=warn ts=2026-02-16T12:06:17.468871025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:06:19.246679435Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T12:06:19.74124493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:06:19.748978327Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:06:20.203854567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:06:20.207483263Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:06:20.32195103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="127 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:06:20.329366368Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:06:20.805075351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:06:20.808333042Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:06:20.844828721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:06:21.207345713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:06:21.211414165Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T12:06:22.468741531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:22.568094711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:06:22.571679342Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:06:23.598861237Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=debug ts=2026-02-16T12:06:23.815622329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:06:24.178750804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:06:24.182451109Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:06:24.904096816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:06:24.908759339Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:06:25.431386025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:06:25.436822006Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:06:26.021161941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:06:26.024685994Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:06:26.3117728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="14 kB" avg_comp="7.2 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:06:26.315302573Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:06:27.468660955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:06:27.468695871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:06:27.468705537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:06:27.468795786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:06:27.95324631Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-02-16T12:06:29.237890317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:06:32.307834333Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T12:06:32.46879842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:06:36.663568887Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=warn ts=2026-02-16T12:06:37.468638111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:39.404284419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:06:39.410612002Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:06:41.017484669Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-02-16T12:06:41.301100124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:06:41.304471822Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:06:41.965853498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:06:41.969719281Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:06:42.469097043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:44.455447466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:06:44.459772453Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:06:45.372468208Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-02-16T12:06:46.147338184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:06:46.151301213Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T12:06:47.468629784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:49.338142446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:06:49.342089847Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:06:49.727593171Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T12:06:49.753846344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:06:49.757621279Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T12:06:50.927087674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:06:52.469007902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:06:53.341168101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:06:53.346546686Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:06:54.082933624Z caller=checkpoint.go:378 msg="writing series" size="776 kB"
level=info ts=2026-02-16T12:06:55.335066036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:06:55.340674375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:06:55.993314184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:06:55.997057927Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:06:57.45276774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:06:57.453238748Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:06:57.468670084Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:06:57.468682549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:06:57.468700035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:06:57.468709712Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T12:06:58.437740777Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-02-16T12:06:59.241228501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=warn ts=2026-02-16T12:07:02.468932369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:02.712579358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:07:02.717090703Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:07:02.798631262Z caller=checkpoint.go:378 msg="writing series" size="9.4 MB"
level=debug ts=2026-02-16T12:07:02.798657003Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=11
level=info ts=2026-02-16T12:07:05.435168038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:07:05.440287836Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:07:06.067032622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="127 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:07:06.074678657Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:07:06.66303358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:07:06.667086057Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:07:07.146934521Z caller=checkpoint.go:378 msg="writing series" size="381 kB"
level=warn ts=2026-02-16T12:07:07.468379745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:08.421135647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:07:08.4252167Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:07:08.425242684Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:07:08.425271906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:07:08.428692748Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:07:08.490271025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:07:08.494141576Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T12:07:11.502303899Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-02-16T12:07:11.548572672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:07:11.562359865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:07:12.469014826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:12.612320324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:07:12.615960492Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:07:14.576696984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:07:14.580436601Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:07:14.98626778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:07:14.988200458Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:07:15.510452835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:33628"
level=debug ts=2026-02-16T12:07:15.856719067Z caller=checkpoint.go:378 msg="writing series" size="314 kB"
level=debug ts=2026-02-16T12:07:16.203728702Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:07:16.840561404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=2 total_comp="14 kB" avg_comp="7.2 kB" total_uncomp="129 kB" avg_uncomp="65 kB" full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:07:16.845411169Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:07:16.964055725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:07:16.967883772Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:07:17.468663382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:18.36240085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:07:18.366201911Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:07:20.212337604Z caller=checkpoint.go:378 msg="writing series" size="812 kB"
level=debug ts=2026-02-16T12:07:20.212360853Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-02-16T12:07:22.468718888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:23.569723557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:07:23.573872991Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:07:24.566028422Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:07:26.046814568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:07:26.05367826Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:07:26.134891711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:07:26.140132408Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:07:27.468452043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:07:27.468465735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:07:27.468497838Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:07:27.468510114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T12:07:28.920948918Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T12:07:29.244339348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:07:29.278984181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:07:29.283470882Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:07:30.664735432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:07:30.668420037Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:07:32.468334009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:07:33.276632802Z caller=checkpoint.go:378 msg="writing series" size="551 kB"
level=info ts=2026-02-16T12:07:33.330777419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:07:33.341756295Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:07:35.20591801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:07:35.211247822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:07:36.307861251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:07:36.311303115Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T12:07:37.469282786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:07:37.636837112Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T12:07:37.947542523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:07:37.951473763Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:07:38.348023954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:07:38.352114716Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T12:07:41.985479701Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=warn ts=2026-02-16T12:07:42.468564376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:43.026561259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:07:43.031073514Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T12:07:45.930968841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54242"
level=debug ts=2026-02-16T12:07:46.340389527Z caller=checkpoint.go:378 msg="writing series" size="9.4 kB"
level=debug ts=2026-02-16T12:07:46.468237988Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:07:47.468792309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:07:50.699894453Z caller=checkpoint.go:378 msg="writing series" size="7.2 MB"
level=debug ts=2026-02-16T12:07:50.699917953Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.9 MB" series=7
level=debug ts=2026-02-16T12:07:50.813022598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T12:07:52.468715788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:53.192695104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:07:53.202178532Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:07:54.486519889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:07:54.490070327Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:07:55.050887756Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-02-16T12:07:55.268437648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="127 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:07:55.275246011Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:07:55.433483051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:07:55.437929434Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:07:56.508867991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:07:56.512746134Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:07:57.101399021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:07:57.104751472Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:07:57.452497168Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:07:57.453540629Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:07:57.469007315Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:07:57.469034514Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:07:57.469045344Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:07:57.469094365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:07:59.121775851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:07:59.124945891Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:07:59.184162142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:07:59.187233151Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T12:07:59.247453525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:07:59.405396981Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=info ts=2026-02-16T12:08:01.620232297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:08:01.623966537Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T12:08:02.468328438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:08:03.760014192Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-02-16T12:08:07.469280544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:08:07.697867305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:08:07.702258324Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:08:08.114974145Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=info ts=2026-02-16T12:08:11.708784116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" idle=1 full=2 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:08:11.712706637Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:08:11.733768474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:08:11.73733582Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T12:08:12.468786529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:08:12.470800397Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB"
level=debug ts=2026-02-16T12:08:12.470833771Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=5
level=info ts=2026-02-16T12:08:13.765934325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:08:13.772844102Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:08:15.162072974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:08:15.165707016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T12:08:16.824871272Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=debug ts=2026-02-16T12:08:17.154110579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:08:17.468441115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:08:18.351479506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:08:18.355238535Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:08:20.587824636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:08:20.591252435Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:08:20.822118164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:08:20.825421779Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:08:21.139474053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:08:21.143027872Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:08:21.179232972Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T12:08:21.798851669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:08:21.802075572Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:08:21.956679763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:08:21.959950248Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T12:08:22.468598717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:08:25.533935099Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T12:08:27.468592367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:08:27.468626839Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:08:27.468634505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:08:27.468632052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:08:27.945997293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:08:27.952119123Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:08:28.99011763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:08:28.993699212Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T12:08:29.249884224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:08:29.445698339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:08:29.448753471Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:08:29.888964409Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-02-16T12:08:32.468317868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:08:34.244226347Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-02-16T12:08:35.198928719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:08:35.205978335Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:08:37.469061242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:08:38.120020477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:08:38.124674672Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:08:38.599145704Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-02-16T12:08:41.910359935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:08:41.914250194Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T12:08:42.469168834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:08:42.953352107Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-02-16T12:08:45.008702498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:08:45.743148819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:08:45.746628363Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:08:47.308138741Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T12:08:47.469031654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:08:50.791080739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:08:50.795071493Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:08:51.663026793Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-02-16T12:08:52.342777605Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:08:52.46866692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:08:53.081048903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:08:53.084479389Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:08:53.454551985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:08:53.457866722Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:08:53.479916593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:08:53.486382627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:08:54.438732902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:08:54.44212153Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:08:54.727087425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:08:54.730455549Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:08:55.158994807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:08:55.162185093Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T12:08:55.162205154Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:08:55.162225739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:08:55.164894848Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:08:56.018262773Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-02-16T12:08:57.452768731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:08:57.452763268Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:08:57.468258793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:08:57.468276707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:08:57.468285349Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:08:57.468292725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T12:08:59.25256372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
ts=2026-02-16T12:09:00.291443557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46260"
level=debug ts=2026-02-16T12:09:00.373058181Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=warn ts=2026-02-16T12:09:02.468931677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:09:03.376692025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:09:03.380674829Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:09:04.72771426Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-02-16T12:09:05.294255649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:09:05.29794641Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:09:06.100737465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:09:06.104598936Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:09:06.839611197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:09:06.842762831Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:09:07.468739163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:09:07.863541074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:09:07.8669166Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:09:09.082089529Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T12:09:09.976687857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:09:09.98054729Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:09:09.980566686Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:09.980593194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:09:09.983404778Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:09:10.249379612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:09:10.253205886Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=error ts=2026-02-16T12:09:10.253227641Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:10.281419908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.5 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:09:10.284769158Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T12:09:10.325918259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:09:10.328898245Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:09:10.328915835Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:10.580756108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="843 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" idle=1 full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:09:10.584037209Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T12:09:12.469287753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:09:13.085829545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:09:13.089206664Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:09:13.437851616Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-02-16T12:09:15.941835283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40268"
level=info ts=2026-02-16T12:09:16.736781741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:09:16.740890027Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:09:16.740914916Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:16.740935449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:09:16.743731417Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:09:17.468843597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:17.792373498Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-02-16T12:09:18.169165243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:09:18.172355657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:09:18.408932994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:09:18.411848792Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:09:18.411868917Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:18.411890075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:09:18.414416125Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:09:21.071499101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:09:21.074675542Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:09:22.146837617Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=debug ts=2026-02-16T12:09:22.146862604Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-02-16T12:09:22.468863534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:23.06995196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:09:24.491246386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.6 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:09:24.494692473Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:09:24.583017778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:09:24.586142867Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:09:26.052238457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:09:26.05525006Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T12:09:26.502353158Z caller=checkpoint.go:378 msg="writing series" size="443 kB"
level=info ts=2026-02-16T12:09:27.211241345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="318 kB" avg_comp="159 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:09:27.215079745Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:09:27.468615112Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:09:27.468651052Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T12:09:27.468649747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:09:27.468662157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:09:27.956736921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:09:27.960004557Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
ts=2026-02-16T12:09:29.255796919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:09:30.856440754Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=info ts=2026-02-16T12:09:31.674415324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:09:31.678346602Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:09:32.468774118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:09:32.801377953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:09:32.804751403Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:09:33.180122654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:09:33.186833499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:09:35.211488068Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=info ts=2026-02-16T12:09:35.840610908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:09:35.845451166Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:09:35.845470341Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:35.845491686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:09:35.848354437Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:09:35.928482123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:09:35.931784676Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:09:36.069735785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:09:36.073098907Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:09:36.073118986Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:36.146311354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:09:36.150103081Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:09:37.469116395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:39.589245956Z caller=checkpoint.go:378 msg="writing series" size="19 MB"
level=debug ts=2026-02-16T12:09:39.589271224Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=4
level=info ts=2026-02-16T12:09:40.35449083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:09:40.357822565Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:09:41.281290663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:09:41.286127023Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:09:41.286147186Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:41.286168947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=2 total_comp="34 kB" avg_comp="17 kB" total_uncomp="222 kB" avg_uncomp="111 kB" full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:09:41.288866353Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:09:41.900644025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:09:41.907360124Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:09:41.90737984Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:41.907400997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:09:41.910152243Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T12:09:42.113318869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T12:09:42.469050879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:43.92144396Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
ts=2026-02-16T12:09:45.520529284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43500"
level=info ts=2026-02-16T12:09:46.100674839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:09:46.104279657Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:09:46.104307637Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:46.104336791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:09:46.107269312Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:09:46.428858012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:09:46.43233Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T12:09:47.468461508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:48.27566612Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T12:09:48.350992893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:09:48.355336635Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:09:50.026397275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:09:51.717976095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="34 kB" avg_comp="11 kB" total_uncomp="223 kB" avg_uncomp="74 kB" idle=1 full=2 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:09:51.72167837Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:09:51.842933504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:09:51.846319071Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T12:09:52.46907982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:52.630385924Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T12:09:52.911429525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:09:52.915278507Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:09:53.24193526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:09:53.245425107Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T12:09:53.95602358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:09:53.959224134Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:09:54.331173629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:09:54.334871772Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:09:56.459843787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:09:56.463467798Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:09:56.463491844Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:09:56.463519415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:09:56.466234301Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:09:56.840140848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:09:56.84344231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T12:09:56.985916592Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T12:09:57.452883903Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:09:57.453259315Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:09:57.468676458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:09:57.468687996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:09:57.468704375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:09:57.468714884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:09:57.896107884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:09:57.899593478Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:09:58.765200671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:09:58.768725214Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
ts=2026-02-16T12:09:59.258954921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:10:00.224929015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:10:00.228188491Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T12:10:00.296023088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52114"
level=info ts=2026-02-16T12:10:00.70761592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:10:00.711095076Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:10:00.711115478Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:10:00.711136324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:10:00.714153314Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:10:00.962230181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:10:00.96589534Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:10:00.965917327Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:10:01.011660572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:10:01.014982668Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T12:10:01.340554971Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=warn ts=2026-02-16T12:10:02.468320731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:03.712403059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:10:03.716198557Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:10:05.051254804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:10:05.055068378Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:10:05.694981361Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=warn ts=2026-02-16T12:10:07.469298332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:10:10.050903181Z caller=checkpoint.go:378 msg="writing series" size="486 kB"
level=debug ts=2026-02-16T12:10:10.05092927Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-02-16T12:10:10.209025004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:10:10.213398829Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T12:10:12.468648581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:13.852245359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:10:13.856542021Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:10:13.902601853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:10:13.909292294Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:10:14.404756573Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T12:10:15.424835623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:10:15.428526904Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:10:16.520377148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=2 total_comp="332 kB" avg_comp="166 kB" total_uncomp="3.3 MB" avg_uncomp="1.6 MB" full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:10:16.524122443Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T12:10:17.469182311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:17.629564191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:10:17.632947612Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:10:18.296914297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:10:18.300222711Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:10:18.541069064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:10:18.545188825Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:10:18.759830595Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-02-16T12:10:19.296418371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:10:20.431351547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:10:20.434778058Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:10:22.468872747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:10:23.114262792Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T12:10:23.337961495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:10:23.341291427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:10:23.77848884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:10:27.46855569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:10:27.46859553Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:10:27.468606201Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:10:27.46860513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:10:27.469692856Z caller=checkpoint.go:390 msg="flushing series" totalSize="32 kB" series=3
level=info ts=2026-02-16T12:10:27.47310763Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-02-16T12:10:27.474232847Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005242703s
level=info ts=2026-02-16T12:10:28.318364436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:10:28.321661586Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T12:10:28.648862025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:10:28.652272106Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T12:10:29.262241215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T12:10:29.883233218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="34 kB" avg_comp="11 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:10:29.886589545Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
ts=2026-02-16T12:10:30.298799321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57048"
level=warn ts=2026-02-16T12:10:32.469045137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:36.347396507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:10:36.351100681Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:10:37.10118806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:10:37.105381799Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:10:37.105402007Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:10:37.105423761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:10:37.108764942Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T12:10:37.468408293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:37.485550942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:10:37.48889387Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:10:38.805292489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:10:38.808931252Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:10:40.17324104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:10:40.17673696Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:10:42.46926601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:43.125210507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:10:43.129788208Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:10:43.815842838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:10:43.819116798Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T12:10:45.524934326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:54374"
ts=2026-02-16T12:10:45.949553646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54332"
level=warn ts=2026-02-16T12:10:47.468418935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:48.134738919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:10:48.140050052Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:10:48.46233641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:10:52.46891542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:53.320730795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:10:53.324928349Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:10:53.721857366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="338 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:10:53.725056334Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:10:54.147387072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:10:54.153641062Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:10:55.135274746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:10:55.139137745Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:10:56.104639488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:10:56.107693067Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:10:56.201915766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:10:56.205037033Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T12:10:57.452950326Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:10:57.45297841Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:10:57.468381448Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T12:10:57.468397046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:10:57.468417132Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:10:57.468425912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:10:57.468468963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:10:57.468506171Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-02-16T12:10:57.468645626Z caller=checkpoint.go:378 msg="writing series" size="518 B"
level=info ts=2026-02-16T12:10:57.885570108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:10:57.888631467Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:10:57.888651234Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:10:57.888674949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:10:57.891287736Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T12:10:59.265013609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T12:10:59.530129588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:10:59.533500127Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:11:01.824136167Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T12:11:02.468447765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:11:06.179475729Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=info ts=2026-02-16T12:11:06.684611835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:11:06.688433673Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T12:11:07.468439037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:11:10.534410171Z caller=checkpoint.go:378 msg="writing series" size="716 kB"
level=info ts=2026-02-16T12:11:11.024468472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:11:11.028632079Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:11:11.691984634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:11:11.695541535Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:11:11.928112643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:11:11.931321977Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T12:11:12.46923264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:12.475258528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" idle=1 full=2 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:11:12.478677764Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:11:12.478692819Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:11:12.478714091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:11:12.481691984Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:11:13.329236982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:11:13.33094838Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T12:11:13.717856398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="34 kB" avg_comp="11 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:11:13.721146341Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:11:14.676395358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:11:14.67958035Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:11:14.888380341Z caller=checkpoint.go:378 msg="writing series" size="895 kB"
level=debug ts=2026-02-16T12:11:14.88840341Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=5
ts=2026-02-16T12:11:15.952796072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58042"
level=info ts=2026-02-16T12:11:17.081008716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:11:17.084374274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T12:11:17.468617768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:18.665380568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:11:18.66921458Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T12:11:19.243648414Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=debug ts=2026-02-16T12:11:21.647053624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:11:22.469149601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:23.329986643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:11:23.33475394Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:11:23.597888237Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=info ts=2026-02-16T12:11:24.957799417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:11:24.96120924Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:11:27.468568898Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:11:27.468609479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:11:27.468620605Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:11:27.468612923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:27.927440166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:11:27.931404743Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:11:27.952728659Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T12:11:28.979719414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:11:28.983884005Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T12:11:29.268030363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T12:11:29.677634516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:11:29.680761611Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-02-16T12:11:30.30295594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56816"
level=debug ts=2026-02-16T12:11:32.307403041Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T12:11:32.468475609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:33.614732524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:11:33.618031006Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:11:33.78661479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:11:33.790446635Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:11:34.312932932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:11:34.31639023Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T12:11:36.663216076Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=warn ts=2026-02-16T12:11:37.468598046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:37.774206046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:11:37.777559738Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T12:11:41.017370451Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-02-16T12:11:42.469007732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:44.133297287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:11:44.138607279Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:11:45.372215261Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-02-16T12:11:45.500855455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:11:45.50477825Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:11:45.504797863Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:11:45.504838054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:11:45.507708303Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:11:46.662493678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:11:46.665776905Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T12:11:47.469134448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:11:47.879779805Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:11:48.842209661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:11:48.846906855Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:11:48.918065364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:11:48.921232391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:11:48.951219183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:11:49.72745838Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T12:11:49.873816341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:11:49.877388483Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:11:51.627320373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:11:51.630638337Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:11:51.630661355Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:11:51.630683103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:11:51.633441921Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:11:52.469288425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:11:53.395484686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:11:53.402086849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:11:53.40210804Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:11:53.40213261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:11:53.407643521Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:11:54.081901711Z caller=checkpoint.go:378 msg="writing series" size="753 kB"
level=info ts=2026-02-16T12:11:54.817621337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:11:54.821384691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:11:57.154345198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:11:57.157935556Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:11:57.266307199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:11:57.269873031Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:11:57.452379136Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:11:57.453428744Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:11:57.468778907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:11:57.468798446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:11:57.46882079Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:11:57.468864733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:11:58.436829471Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-02-16T12:11:59.271009512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T12:12:02.086277445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:12:02.089881526Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:12:02.346860935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="34 kB" avg_comp="11 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:12:02.350384387Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T12:12:02.469083279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:12:02.798527434Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-02-16T12:12:02.798552235Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=11
level=info ts=2026-02-16T12:12:03.791176651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:12:03.794336722Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T12:12:04.041929569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:12:04.048209919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:12:04.487618983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:12:04.49100956Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:12:05.27705903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:12:05.280303392Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:12:05.446777618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:12:05.450031297Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:12:05.450050637Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:12:05.450072192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:12:05.45265368Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T12:12:07.146414069Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=warn ts=2026-02-16T12:12:07.468446192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:08.830219236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:12:08.833881951Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:12:11.501554952Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-02-16T12:12:12.469014845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T12:12:15.531507215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52152"
level=debug ts=2026-02-16T12:12:15.856165156Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=info ts=2026-02-16T12:12:16.116543495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:12:16.121120157Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:12:17.324651866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:12:17.328825351Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:12:17.405229331Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:12:17.468595808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:20.027861062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:12:20.031827402Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:12:20.211659206Z caller=checkpoint.go:378 msg="writing series" size="815 kB"
level=debug ts=2026-02-16T12:12:20.211683952Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-02-16T12:12:22.469254798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:23.061853568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:12:23.065153166Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:12:23.638387293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:12:24.56612111Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:12:25.582077744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="7.5 kB" avg_uncomp="7.5 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:12:25.585268643Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:12:27.468780426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:27.468790615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:12:27.468832506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:12:27.468841366Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:12:27.915202198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:12:27.918927624Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T12:12:28.921251863Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T12:12:29.274638111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T12:12:30.793742368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:12:30.800329342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:12:32.469110477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:32.938830866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:12:32.942279226Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:12:32.942300552Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:12:32.942321406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:12:32.945184269Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T12:12:33.276429535Z caller=checkpoint.go:378 msg="writing series" size="755 kB"
level=info ts=2026-02-16T12:12:33.674814034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:12:33.678084774Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:12:36.960501334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:12:36.964237968Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:12:37.211855722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:12:37.215140859Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:12:37.329471008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:12:37.332762715Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:12:37.388130373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:12:37.391096125Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:12:37.469114056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:12:37.630391514Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:12:40.549350756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:12:40.552655719Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:12:40.814284529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:12:40.817408207Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:12:41.985519406Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=warn ts=2026-02-16T12:12:42.469163289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:44.283234242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="35 kB" avg_comp="12 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:12:44.287052622Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:12:45.260660807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:12:45.264139227Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T12:12:45.959630201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41920"
level=info ts=2026-02-16T12:12:46.005854164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:12:46.009932233Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T12:12:46.339917497Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-02-16T12:12:47.252397783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:12:47.257870477Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:12:47.379841596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:12:47.383539531Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T12:12:47.469154779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:12:50.698034327Z caller=checkpoint.go:378 msg="writing series" size="8.0 MB"
level=debug ts=2026-02-16T12:12:50.698057381Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.9 MB" series=7
level=debug ts=2026-02-16T12:12:52.327468391Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:12:52.46856994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:12:52.784097481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:12:52.788961904Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:12:55.050004703Z caller=checkpoint.go:378 msg="writing series" size="426 kB"
level=info ts=2026-02-16T12:12:56.820649209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:12:56.823958147Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:12:57.452975618Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:12:57.453350669Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:12:57.468684307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:12:57.468706215Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:12:57.468714424Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:12:57.468785752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T12:12:59.277855433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:12:59.40472419Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-02-16T12:13:02.468977266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:03.110711116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:13:03.115497232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T12:13:03.760068465Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T12:13:03.914502732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:13:03.918220302Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:13:04.127855326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:13:04.131369806Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:13:04.235582151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:13:04.239301192Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T12:13:07.469102447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:13:08.114425419Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-02-16T12:13:09.970020864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:13:09.973369173Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:13:12.278374215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:13:12.281649363Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-02-16T12:13:12.469092615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:13:12.470170252Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T12:13:12.470187087Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=5
level=info ts=2026-02-16T12:13:14.643647421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:13:14.647116891Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T12:13:15.480358901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:13:15.536224168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:46630"
level=info ts=2026-02-16T12:13:15.723131342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:13:15.72669744Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:13:16.689671588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:13:16.693626019Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:13:16.823978364Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T12:13:17.37794967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="923 B" avg_comp="923 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:13:17.38165016Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T12:13:17.468459189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:18.973673556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:13:18.980281509Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:13:19.698535885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T12:13:21.179155778Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-02-16T12:13:22.469405381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:22.600729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" idle=1 full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:13:22.604238308Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:13:24.151369333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:13:24.154606265Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:13:24.758390846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:13:24.76318724Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:13:24.839296626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="35 kB" avg_comp="12 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:13:24.842514855Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:13:25.534585573Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T12:13:27.469222645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:13:27.469254176Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:13:27.469262544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:13:27.469256772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:27.635641576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:13:27.639728935Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:13:28.464045589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:13:28.467583005Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T12:13:29.280850991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:13:29.889069126Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T12:13:29.916113968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:13:29.919707002Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:13:31.075512583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:13:31.078715381Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:13:32.468726981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:32.889096324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" idle=1 full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:13:32.892498795Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:13:34.210225383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:13:34.214046371Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:13:34.214068365Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:13:34.214091608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:13:34.216752465Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:13:34.243934852Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-02-16T12:13:37.46911417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:13:38.598398249Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-02-16T12:13:40.834730791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:13:40.837747145Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:13:41.781723015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:13:41.785288208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T12:13:42.468747936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:13:42.952877977Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-02-16T12:13:44.701880509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:13:44.705468633Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T12:13:45.538846583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:38228"
level=debug ts=2026-02-16T12:13:47.308350584Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T12:13:47.4692113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:13:49.350626346Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T12:13:51.663007428Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-02-16T12:13:52.004255054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T12:13:52.469316608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:52.538496101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:13:52.541975503Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:13:53.565003954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:13:53.56842581Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:13:55.944993971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:13:55.950015717Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:13:56.018130278Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=debug ts=2026-02-16T12:13:57.452609076Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:13:57.453655046Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:13:57.468985752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:13:57.469009858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:13:57.469018495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:13:57.469019498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:13:59.17288511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:13:59.17623937Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T12:13:59.283090561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
ts=2026-02-16T12:13:59.284132489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=error ts=2026-02-16T12:13:59.286377507Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T12:14:00.372590076Z caller=checkpoint.go:378 msg="writing series" size="397 kB"
level=debug ts=2026-02-16T12:14:00.372613313Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-02-16T12:14:01.97465766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:14:01.978042098Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:14:02.469188199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:04.727469439Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=info ts=2026-02-16T12:14:05.467731177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:14:05.472499012Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:14:05.492505669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:14:05.496483066Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:14:05.618677417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:14:05.622050297Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T12:14:07.468335569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:14:08.504827194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.5 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:14:08.508249927Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T12:14:09.010611975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="35 kB" avg_comp="12 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:14:09.013799327Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:14:09.082137834Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T12:14:09.95986185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:14:09.962902617Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:14:10.661704851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:14:10.668504535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:14:12.469297232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:13.437491107Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-02-16T12:14:15.054784009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:14:15.058702811Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T12:14:15.542155719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:48012"
level=info ts=2026-02-16T12:14:15.59361847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="843 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:14:15.597197926Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:14:16.91112013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:14:16.914453535Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:14:16.968683302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:14:16.971846853Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:14:17.468501819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:17.791487521Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-02-16T12:14:18.240095476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:14:18.243247011Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:14:19.84685521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:14:19.85066763Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:14:21.818273131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.6 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:14:21.822347937Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:14:22.14650862Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-02-16T12:14:22.468878183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:22.545243674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:14:26.012472209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:14:26.016892449Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:14:26.501408534Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=info ts=2026-02-16T12:14:27.468313359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:14:27.468344494Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:14:27.468353199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:14:27.468377066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T12:14:29.286923245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=debug ts=2026-02-16T12:14:30.856913538Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=warn ts=2026-02-16T12:14:32.469072304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:14:32.504242303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:14:32.508200386Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:14:32.92742322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:14:32.931009887Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:14:33.279081128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="14 kB" avg_comp="4.8 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:14:33.282373653Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:14:35.211379769Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=info ts=2026-02-16T12:14:35.657053869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:14:35.661063427Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T12:14:37.46831305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:38.866595041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:14:38.868174416Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:14:39.582316758Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-02-16T12:14:39.582341463Z caller=checkpoint.go:390 msg="flushing series" totalSize="21 MB" series=9
level=info ts=2026-02-16T12:14:39.871485129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:14:39.874979384Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:14:42.423127068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:14:42.42636843Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T12:14:42.469103697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:43.920566189Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T12:14:46.329380731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:14:46.334349731Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:14:47.165922261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:14:47.169192228Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:14:47.46829966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:48.275706045Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T12:14:50.612336775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:14:50.616941416Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:14:52.182680192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:14:52.186065017Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T12:14:52.469133986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:14:52.630456725Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=debug ts=2026-02-16T12:14:52.712721946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:14:52.712830582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:14:52.720002637Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:14:53.875383834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="35 kB" avg_comp="12 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:14:53.879115823Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:14:54.653848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:14:54.657133568Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:14:54.685149963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:14:54.688503971Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:14:56.985792875Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T12:14:57.452769824Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:14:57.453106297Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:14:57.468464594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:14:57.468493128Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:14:57.468502139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:14:57.468523936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:14:58.972171039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:14:58.976925423Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T12:14:59.289962289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:14:59.859627542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:14:59.867052143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:15:01.340385638Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=info ts=2026-02-16T12:15:02.393358622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:15:02.398171136Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T12:15:02.469177279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:03.203334627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:15:03.206833457Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:15:04.030660815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="332 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:15:04.034604751Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:15:04.351722067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:15:04.35511092Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T12:15:05.694910845Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-02-16T12:15:07.468424812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:07.547777425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:15:07.556045524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:15:09.650561721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=3 total_comp="1.0 kB" avg_comp="344 B" total_uncomp="1.0 kB" avg_uncomp="343 B" full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:15:09.65413534Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:15:10.050510526Z caller=checkpoint.go:378 msg="writing series" size="404 kB"
level=warn ts=2026-02-16T12:15:12.469118749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:15:14.404611303Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T12:15:15.508218988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:15:15.512263918Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-02-16T12:15:16.898380807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:15:16.901886991Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:15:17.364266331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:15:17.368081421Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:15:17.469212427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:18.698372085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:15:18.701591983Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:15:18.759847664Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T12:15:19.363442918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="16 kB" avg_comp="5.4 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:15:19.367169103Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:15:19.367190933Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:15:19.367214825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="16 kB" avg_comp="5.4 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:15:19.370246041Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T12:15:22.469182183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:15:23.056926827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:15:23.11415125Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=debug ts=2026-02-16T12:15:23.114172794Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-02-16T12:15:26.241688593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:15:26.428857856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:15:26.433607077Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:15:27.248391132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:15:27.251861002Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T12:15:27.468489604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:15:27.468519748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:15:27.468529127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:15:27.468592475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:27.474102835Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-02-16T12:15:27.475384986Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006767917s
level=info ts=2026-02-16T12:15:28.420726513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:15:28.424326541Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T12:15:28.424344972Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:15:28.424366443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:15:28.427420923Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:15:28.839180463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:15:28.843012476Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-02-16T12:15:29.292425119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=warn ts=2026-02-16T12:15:32.468538082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:32.798258433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:15:32.798627804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:15:32.803546955Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:15:32.803551664Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:15:32.803565137Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:15:32.803588418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:15:32.806745001Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:15:35.763214096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="16 kB" avg_comp="5.4 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:15:35.766685404Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T12:15:37.468477958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:38.691481254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:15:38.697423027Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:15:40.825456392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:15:40.828836926Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:15:42.469160449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:42.580299452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:15:42.5835705Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:15:45.605009449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:15:45.61024704Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:15:46.355760366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:15:46.362242876Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:15:46.733100914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:15:46.736973405Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T12:15:47.468970121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:47.91981838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:15:47.923547017Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:15:47.923572093Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:15:47.923600491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:15:47.926632371Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:15:48.2315048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:15:48.235221573Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:15:48.298564524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:15:48.302395821Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:15:49.369393703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:15:49.372469553Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:15:50.990245531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="35 kB" avg_comp="12 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:15:50.99364439Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:15:50.993665182Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:15:50.993691275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=3 total_comp="35 kB" avg_comp="12 kB" total_uncomp="223 kB" avg_uncomp="74 kB" full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:15:50.996465595Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:15:51.151246634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:15:51.154484705Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:15:52.46849995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:53.065610119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:15:53.071974Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:15:53.306512977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="363 B" total_uncomp="1.4 kB" avg_uncomp="342 B" idle=1 full=3 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:15:53.309744763Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:15:57.452953707Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:15:57.45296476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=debug ts=2026-02-16T12:15:57.45482256Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T12:15:57.454880584Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968048 n=1
level=debug ts=2026-02-16T12:15:57.454924164Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968048 n=1
level=info ts=2026-02-16T12:15:57.468204612Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T12:15:57.46825335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:15:57.468281602Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:15:57.468291022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:15:57.468309067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:15:57.46853227Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-02-16T12:15:57.468725532Z caller=checkpoint.go:378 msg="writing series" size="907 B"
level=debug ts=2026-02-16T12:15:57.946638011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:15:58.526636717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:15:58.529838418Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
ts=2026-02-16T12:15:59.295301074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:16:01.139154104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:16:01.390326585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:16:01.39371718Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:16:01.824006358Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T12:16:02.398659374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:16:02.402282073Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:16:02.402302032Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:02.402321905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:16:02.404954524Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T12:16:02.468987508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:16:06.178997765Z caller=checkpoint.go:378 msg="writing series" size="267 kB"
level=info ts=2026-02-16T12:16:06.683614246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:16:06.688513051Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:16:06.688534642Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:06.688558456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:16:06.691510262Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-02-16T12:16:07.469073331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:07.800242106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.7 kB" total_uncomp="223 kB" avg_uncomp="56 kB" idle=1 full=3 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:16:07.805687181Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:16:10.533864368Z caller=checkpoint.go:378 msg="writing series" size="763 kB"
level=warn ts=2026-02-16T12:16:12.468460561Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:12.646944615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="16 kB" avg_comp="5.4 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:16:12.65207841Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:16:14.408958334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:16:14.41236499Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:16:14.412385433Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:14.412409214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:16:14.414887494Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:16:14.659316486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:16:14.662560562Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:16:14.6625786Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:14.712765365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:16:14.716053702Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T12:16:14.889007087Z caller=checkpoint.go:378 msg="writing series" size="896 kB"
level=debug ts=2026-02-16T12:16:14.889030077Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=5
level=info ts=2026-02-16T12:16:16.149536347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:16:16.152783643Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:16:16.651093735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:16:16.654565119Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:16:16.766643289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="330 B" avg_comp="330 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:16:16.769841461Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:16:16.769858914Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:16.769879101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:16:16.772725074Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:16:17.468933252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:17.871493037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:16:17.875195041Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:16:19.243738536Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T12:16:19.712087985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:16:19.715448134Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:16:21.698098289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:16:21.701290192Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:16:22.450595058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:16:22.453738198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T12:16:22.453756074Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:22.453784961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:16:22.4603226Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T12:16:22.468434613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:23.125552101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:16:23.129378023Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:16:23.598729593Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=debug ts=2026-02-16T12:16:25.794319407Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:16:26.39449048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:16:26.40022386Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:16:27.468675038Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:16:27.46869602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:16:27.468711595Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:16:27.468723043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:16:27.469800878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:16:27.472965401Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:16:27.953695976Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T12:16:28.353923231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:16:28.357135361Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:16:28.357154857Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:28.357175549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:16:28.359698394Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:16:28.817046173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:16:28.820179353Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:16:29.1572226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:16:29.160676147Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T12:16:29.298182721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:16:30.328348617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:16:30.331543834Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T12:16:32.307873237Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T12:16:32.413317379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:16:32.416822029Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:16:32.469163803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:33.503356422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.6 kB" avg_comp="4.8 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:16:33.506911834Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:16:34.671344599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:16:34.675518175Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:16:35.186919325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:16:35.190251683Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:16:35.190270225Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:16:35.190291779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:16:35.193136921Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:16:35.279474056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.7 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:16:35.282797445Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:16:36.66266716Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-02-16T12:16:36.788922855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:16:36.793866921Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T12:16:37.469251165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:40.232239081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:16:40.236030816Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:16:41.017615025Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T12:16:41.328739183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:16:41.332373533Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:16:41.724649535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:16:41.731313006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T12:16:42.468786484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:44.247783235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:16:44.251405189Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:16:45.372397265Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-02-16T12:16:46.575764036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:16:46.579298874Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:16:47.030592253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:16:47.034238972Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:16:47.468373981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:48.285008371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:16:48.291532583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:16:49.727433668Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-02-16T12:16:52.468502572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:16:52.619879281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T12:16:54.082983005Z caller=checkpoint.go:378 msg="writing series" size="797 kB"
level=info ts=2026-02-16T12:16:56.468918118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:16:56.472138035Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T12:16:56.87640347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:16:57.452855336Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:16:57.45286182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=warn ts=2026-02-16T12:16:57.469248941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:16:57.469253687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:16:57.469274681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:16:57.469285003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:16:57.756850742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:16:57.760720197Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:16:58.436779804Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-02-16T12:16:58.436823423Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T12:16:58.945667984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:16:58.948933041Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T12:16:59.300952416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:16:59.832853754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:16:59.836252304Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T12:17:00.322553023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49780"
level=info ts=2026-02-16T12:17:00.678383866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="348 B" avg_comp="348 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:17:00.681523796Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:17:02.201388521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=3 total_comp="16 kB" avg_comp="5.4 kB" total_uncomp="144 kB" avg_uncomp="48 kB" full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:17:02.204769427Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-02-16T12:17:02.469103586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:17:02.796540599Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-02-16T12:17:02.796562103Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=1
level=info ts=2026-02-16T12:17:03.66169496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:17:03.668820523Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:17:05.648406544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:17:05.651782577Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:17:07.147068418Z caller=checkpoint.go:378 msg="writing series" size="461 kB"
level=warn ts=2026-02-16T12:17:07.469317251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:07.731543377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:17:07.734721008Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:17:07.73474404Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:17:07.734771342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:17:07.737419175Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:17:08.604085447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="9.8 kB" avg_comp="4.9 kB" total_uncomp="46 kB" avg_uncomp="23 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:17:08.607220906Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T12:17:11.501657291Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-02-16T12:17:12.468678788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:12.900410327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:17:12.905106516Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:17:13.571739883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="941 B" avg_comp="941 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:17:13.575591604Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:17:15.626507785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:17:15.631377805Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:17:15.856422485Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
ts=2026-02-16T12:17:15.981279221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60084"
level=debug ts=2026-02-16T12:17:16.669833581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:17:17.468562791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:18.069550377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:17:18.073105499Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:17:18.560011309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:17:18.563883392Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:17:19.699859972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:17:19.706055315Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T12:17:20.211530535Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T12:17:20.211555935Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=4
level=info ts=2026-02-16T12:17:21.383785282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:17:21.387766298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:17:21.387786256Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:17:21.387825432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:17:21.391268026Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:17:21.440367049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.8 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:17:21.443365829Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T12:17:22.468355148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:23.768843892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:17:23.772243401Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:17:23.772264029Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:17:23.772283667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:17:23.774878099Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:17:24.259223383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:17:24.262495117Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T12:17:24.566413977Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:17:27.469228935Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:17:27.469255601Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-02-16T12:17:27.469244135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:27.469264179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:17:28.917556089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:17:28.920320798Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
level=error ts=2026-02-16T12:17:28.921731139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:17:28.921746921Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:17:28.921771305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:17:28.925076065Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T12:17:29.304312123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:17:29.511246937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:17:29.518669257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:17:31.228537089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:17:31.231687873Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:17:31.506743282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:17:31.51085377Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T12:17:32.468919592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:17:33.27686983Z caller=checkpoint.go:378 msg="writing series" size="746 kB"
level=info ts=2026-02-16T12:17:36.619082262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:17:36.62256222Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T12:17:37.469289694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:17:37.630587317Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:17:38.102924448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:17:38.108092537Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:17:38.490391979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:17:38.494252676Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:17:39.929257987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:17:39.932588963Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:17:39.933580183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:17:39.936747883Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:17:41.001246191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" idle=1 full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:17:41.004534861Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:17:41.390917156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T12:17:41.985643917Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=warn ts=2026-02-16T12:17:42.468482994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:17:46.340085346Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T12:17:47.46862653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:48.650049613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:17:48.655408464Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=debug ts=2026-02-16T12:17:50.699368802Z caller=checkpoint.go:378 msg="writing series" size="8.5 MB"
level=debug ts=2026-02-16T12:17:50.699400256Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.4 MB" series=7
level=info ts=2026-02-16T12:17:50.945261499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:17:50.948737671Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:17:51.742162972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:17:51.749742667Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:17:52.016024922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:17:52.019271553Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T12:17:52.468944877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:17:52.837918438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-02-16T12:17:52.991618752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:17:52.99497426Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:17:55.049869126Z caller=checkpoint.go:378 msg="writing series" size="331 kB"
level=info ts=2026-02-16T12:17:55.393553613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:17:55.396963935Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T12:17:56.980851162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" idle=1 full=3 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:17:56.984005852Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:17:57.452774444Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:17:57.45315005Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:17:57.468413265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:17:57.468440197Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:17:57.468449354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:17:57.468458376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:17:59.127755362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" idle=1 full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:17:59.130885992Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T12:17:59.307659222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:17:59.404432146Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-02-16T12:17:59.648355912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="523 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" idle=1 full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:17:59.651386547Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:18:00.559332935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:18:00.563417204Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:18:01.53240634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:18:01.535640785Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T12:18:02.4685782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:18:03.759878633Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T12:18:03.988604367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:18:03.992636701Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:18:07.468575574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:18:08.114465824Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-02-16T12:18:08.21772912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:18:08.221898627Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:18:09.147850501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:18:09.151097431Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:18:09.151116881Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:18:09.151138393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:18:09.153743935Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:18:12.468896256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:18:12.470037224Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T12:18:12.470052936Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=5
level=info ts=2026-02-16T12:18:13.673730105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.8 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:18:13.677322353Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=debug ts=2026-02-16T12:18:14.082080343Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:18:16.745580395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:18:16.749132528Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:18:16.749152713Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:18:16.749174988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:18:16.752086144Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T12:18:16.824250516Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-02-16T12:18:17.116238449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:18:17.119519388Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:18:17.119540104Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:18:17.11956094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:18:17.122323306Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:18:17.335872839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:18:17.342950872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T12:18:17.468708228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:18.528413587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:18:18.532169988Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:18:21.179195522Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-02-16T12:18:22.468409811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:22.789220069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:18:22.794940164Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:18:24.93635169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:18:24.939985486Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:18:25.534199603Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T12:18:26.461916454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:18:26.465720531Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T12:18:27.468566737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:27.468563279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:18:27.468599804Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:18:27.468610865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T12:18:27.851360365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:18:27.85512785Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T12:18:29.310381287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:18:29.38987365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:18:29.397470424Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:18:29.397494576Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:18:29.39752291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:18:29.403905337Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:18:29.888637559Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T12:18:29.964088776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:18:29.967483934Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:18:31.541739372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:18:31.544944268Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-02-16T12:18:32.468948741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:33.286934979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:18:33.29006948Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:18:34.027600042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:18:34.030954348Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:18:34.243682709Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
level=info ts=2026-02-16T12:18:34.951060668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:18:34.954231658Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:18:35.852109397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=info ts=2026-02-16T12:18:35.854751928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:18:35.855654569Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=error ts=2026-02-16T12:18:35.857387855Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:18:37.041854925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:18:37.045007762Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:18:37.468430627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:18:38.598424161Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-02-16T12:18:42.468611091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:42.508713052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:18:42.515668771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:18:42.761167603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:18:42.762924135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:18:42.953409794Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-02-16T12:18:44.554867004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:18:44.562488429Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T12:18:47.307858591Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T12:18:47.469013506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:48.316704299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:18:48.320111807Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:18:49.118483485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:18:49.121958031Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:18:49.900252069Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:18:51.593962184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:18:51.597463381Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:18:51.662896807Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T12:18:52.468586002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:52.866376601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:18:52.870516953Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:18:56.017541153Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=info ts=2026-02-16T12:18:56.089108046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:18:56.092558241Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:18:56.825551063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:18:56.829032198Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:18:57.452068817Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:18:57.453125799Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:18:57.468402226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:18:57.468430642Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:18:57.468443871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:18:57.468487807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:18:58.216384517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.8 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:18:58.219836843Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:18:58.520818688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:18:58.526680944Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T12:18:59.313233315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:18:59.53324229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:18:59.537260298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:19:00.372497446Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=debug ts=2026-02-16T12:19:00.372523Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-02-16T12:19:02.468485397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:19:02.737747188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:19:02.741417474Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:19:02.741439459Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:19:02.741460366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:19:02.744312342Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:19:04.294712322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:19:04.29814472Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T12:19:04.727488345Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-02-16T12:19:07.46861933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:19:09.081674475Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-02-16T12:19:09.86501123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:19:09.86883029Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:19:10.297912109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:19:10.301931698Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:19:12.468934529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:19:12.550160419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="333 kB" avg_comp="111 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:19:12.553686927Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T12:19:13.436955065Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-02-16T12:19:14.18585627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:19:14.189256859Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:19:15.159225601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:19:15.162623622Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:19:16.100481104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:19:16.103657883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:19:17.147825128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:19:17.154925653Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:19:17.468700564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:19:17.79138828Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-02-16T12:19:18.580336871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:19:18.583571118Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:19:22.146638182Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-02-16T12:19:22.469271462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:19:24.588773684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:19:24.59347326Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:19:25.25550855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:19:25.85674754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:19:25.860321019Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:19:26.001998937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:19:26.005487878Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T12:19:26.501756187Z caller=checkpoint.go:378 msg="writing series" size="396 kB"
level=info ts=2026-02-16T12:19:27.468516132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T12:19:27.468523814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:19:27.468545492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:19:27.468553855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-02-16T12:19:29.316545912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=info ts=2026-02-16T12:19:29.875663308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:19:29.879190757Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:19:30.011514853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:19:30.014643379Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:19:30.856114555Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=info ts=2026-02-16T12:19:31.093975872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:19:31.09702101Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:19:31.174223677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:19:31.177381531Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:19:32.46862778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:19:34.471830937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:19:34.475267877Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T12:19:35.211321955Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=debug ts=2026-02-16T12:19:35.211346923Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-02-16T12:19:36.352062261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:19:36.355488435Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:19:37.468379483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:19:38.556785894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:19:38.560166712Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:19:39.584212734Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-02-16T12:19:39.584236299Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=1
level=debug ts=2026-02-16T12:19:41.109390713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-02-16T12:19:42.468491982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:19:42.748997796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:19:42.755849564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T12:19:43.920865952Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T12:19:44.542754816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:19:44.54586564Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T12:19:45.561456748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55912"
level=info ts=2026-02-16T12:19:45.739438959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:19:45.74529817Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:19:47.298543098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:19:47.302424155Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T12:19:47.469130848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:19:48.275460623Z caller=checkpoint.go:378 msg="writing series" size="763 B"
level=info ts=2026-02-16T12:19:49.108853317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:19:49.112112978Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:19:51.118936298Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T12:19:52.469253769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:19:52.632146374Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-02-16T12:19:53.08572911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.8 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:19:53.091016511Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:19:53.567917916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:19:53.571099322Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:19:54.32703054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="393 B" avg_comp="393 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:19:54.330257591Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:19:54.922020572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:19:54.926360954Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:19:56.154877517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:19:56.158079742Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:19:56.98557583Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T12:19:57.452772964Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T12:19:57.453151307Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:19:57.468411289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:19:57.468431538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:19:57.468442507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:19:57.468486252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T12:19:59.319574016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:20:01.340053671Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=warn ts=2026-02-16T12:20:02.46898427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:03.105939488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.6 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:20:03.109574813Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:20:03.109596824Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:20:03.109622175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:20:03.112457777Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:20:03.292079916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:20:03.298941213Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T12:20:05.695482352Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=warn ts=2026-02-16T12:20:07.469258335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:09.501363666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:20:09.505184306Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:20:10.049931808Z caller=checkpoint.go:378 msg="writing series" size="411 kB"
level=info ts=2026-02-16T12:20:11.511697717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="336 kB" avg_comp="112 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:20:11.516324059Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T12:20:12.468600273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:14.267098902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:20:14.271039017Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T12:20:14.404287177Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-02-16T12:20:14.515829772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:20:14.519117426Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:20:14.976448771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:20:14.979909566Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:20:15.283885383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:20:15.761665696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:20:15.765033269Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:20:15.778103477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:20:15.780926105Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:20:16.282379505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:20:16.285536239Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T12:20:17.468296548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:20:17.877957114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:20:18.760003289Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T12:20:21.571466807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:20:21.574946664Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T12:20:22.46872055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:20:23.114136485Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=debug ts=2026-02-16T12:20:23.114174111Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-02-16T12:20:24.058264784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:20:24.062528632Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:20:25.350223861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:20:25.354167092Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:20:26.792114994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:20:26.800553676Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:20:27.468740864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:20:27.468777818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:20:27.468789445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:20:27.468829604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:27.474365994Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-02-16T12:20:27.475782516Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007104981s
level=info ts=2026-02-16T12:20:27.797855514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:20:27.80095593Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:20:28.000189933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:20:28.003393839Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
ts=2026-02-16T12:20:29.322741415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=warn ts=2026-02-16T12:20:32.469045753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:34.937706127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:20:34.941465832Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:20:36.911727734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:20:36.915197062Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:20:37.469068518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:37.624386155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:20:37.62772483Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:20:38.877692441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.8 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:20:38.881380213Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:20:39.018728572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:20:39.022466651Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:20:40.376695187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="363 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:20:40.380205962Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T12:20:42.468555953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:43.155845441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:20:43.159501161Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:20:44.666254229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:20:44.67246691Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T12:20:47.469143927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:48.541616439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:20:48.544894253Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:20:49.673210904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:20:49.676594709Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:20:50.603131505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:20:50.957011022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:20:50.964595914Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T12:20:52.46891756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:53.871773978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:20:53.875462287Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:20:57.421460831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:20:57.424585244Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:20:57.452564012Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:20:57.453604199Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:20:57.468903475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:20:57.468921943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:20:57.468924778Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T12:20:57.468931393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:20:57.468933258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:20:57.469051764Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-02-16T12:20:57.469211265Z caller=checkpoint.go:378 msg="writing series" size="909 B"
ts=2026-02-16T12:20:59.325474586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:21:01.824237831Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-02-16T12:21:02.468341887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:03.72043055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="336 kB" avg_comp="112 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:21:03.724047347Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:21:03.861632179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:21:03.864706116Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:21:04.99950932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:21:05.002687444Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:21:06.172554927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:21:06.179732048Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=info ts=2026-02-16T12:21:06.85940864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:21:06.862664843Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:21:06.998920527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:21:07.002101419Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T12:21:07.468776473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:09.146869088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:21:09.150089408Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:21:10.534200984Z caller=checkpoint.go:378 msg="writing series" size="605 kB"
level=warn ts=2026-02-16T12:21:12.468514069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:21:14.890126703Z caller=checkpoint.go:378 msg="writing series" size="898 kB"
level=debug ts=2026-02-16T12:21:14.89015127Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-02-16T12:21:17.468988034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:17.504125144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:21:17.508302838Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:21:17.512364702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:21:17.51941167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T12:21:18.213097016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:21:18.216767146Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:21:18.301246668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:21:18.304491544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T12:21:19.082737296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:21:19.085928611Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T12:21:19.244271104Z caller=checkpoint.go:378 msg="writing series" size="1.2 kB"
level=info ts=2026-02-16T12:21:20.265263957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:21:20.268587875Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:21:21.455646757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:21:21.458772057Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=warn ts=2026-02-16T12:21:22.469241049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:21:23.598712292Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=debug ts=2026-02-16T12:21:24.035044544Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:21:25.710106689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="523 B" avg_comp="261 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:21:25.713964289Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:21:27.469114577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:21:27.46914426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:21:27.469154788Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:21:27.46914934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:27.915078743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:21:27.919830191Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-02-16T12:21:27.954159185Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-02-16T12:21:28.565006053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:21:28.567977561Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T12:21:29.3281347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:21:32.308275536Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-02-16T12:21:32.468906763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:33.170865074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=4 total_comp="35 kB" avg_comp="8.8 kB" total_uncomp="223 kB" avg_uncomp="56 kB" full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:21:33.174313453Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:21:33.335734908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:21:33.342216468Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:21:33.342238369Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:21:33.342265581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:21:33.348180519Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:21:35.160653018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:21:35.163865875Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:21:36.317756407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:21:36.321094703Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:21:36.663089739Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=warn ts=2026-02-16T12:21:37.468685994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:21:41.01787495Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=warn ts=2026-02-16T12:21:42.468563753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:45.023834916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:21:45.028020767Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:21:45.373087082Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
ts=2026-02-16T12:21:45.570657411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40046"
level=info ts=2026-02-16T12:21:46.419242522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:21:46.425475116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:21:46.841714283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:21:46.845216184Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T12:21:47.469354249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:21:47.820947855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:21:47.82502221Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:21:49.042080255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:21:49.045479761Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:21:49.425400946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:21:49.428941111Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:21:49.428962543Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:21:49.428983262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:21:49.431694948Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=debug ts=2026-02-16T12:21:49.727783405Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=info ts=2026-02-16T12:21:50.775225259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:21:50.781701865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:21:50.802714353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="336 kB" avg_comp="112 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:21:50.806127286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:21:51.037110594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="619 B" avg_comp="619 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:21:51.040536113Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T12:21:51.040554921Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:21:51.040575991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:21:51.043515412Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:21:52.468449849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:21:52.470230395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:21:54.047685732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-02-16T12:21:54.083086925Z caller=checkpoint.go:378 msg="writing series" size="618 kB"
level=info ts=2026-02-16T12:21:54.385751578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="546 B" avg_comp="546 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:21:54.388950625Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:21:54.388970744Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:21:54.388991665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:21:54.391510673Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:21:57.452468984Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:21:57.453531803Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:21:57.468731932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:21:57.468756968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:21:57.468765373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:21:57.46879409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:21:58.437461857Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-02-16T12:21:59.330456654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:22:00.341429166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35420"
level=info ts=2026-02-16T12:22:01.918623084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:22:01.923987626Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T12:22:02.237822438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:22:02.241233242Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-02-16T12:22:02.468768591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:22:02.797176551Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-02-16T12:22:02.797200553Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=11
level=info ts=2026-02-16T12:22:04.094287976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:22:04.098144188Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:22:04.990858961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:22:04.99454192Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:22:06.419629915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:22:06.423131389Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:22:07.147082415Z caller=checkpoint.go:378 msg="writing series" size="471 kB"
level=warn ts=2026-02-16T12:22:07.46913472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:08.520724435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:22:08.525585172Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:22:09.439029323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:22:09.443298432Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:22:09.443318732Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:09.443341169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:22:09.446101446Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:22:09.921476857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:22:09.925063157Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:22:10.884010373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:22:10.890114929Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T12:22:11.502314249Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-02-16T12:22:12.004692409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:22:12.011096923Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T12:22:12.469073285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:13.402034316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:22:13.405364519Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T12:22:13.61010767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:22:13.613367642Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:22:13.613386081Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:13.613409581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:22:13.616000136Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T12:22:14.567792992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:22:14.571179494Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T12:22:15.857446832Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
level=warn ts=2026-02-16T12:22:17.469201627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:22:20.212117186Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-02-16T12:22:20.212141853Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-02-16T12:22:20.219587394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:22:20.224089528Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=info ts=2026-02-16T12:22:21.171526051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="35 kB" avg_comp="7.1 kB" total_uncomp="224 kB" avg_uncomp="45 kB" idle=1 full=4 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:22:21.175142902Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:22:21.175163789Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:21.175186722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="35 kB" avg_comp="7.1 kB" total_uncomp="224 kB" avg_uncomp="45 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:22:21.178549218Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:22:22.401307504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:22:22.405542519Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-02-16T12:22:22.46918261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:24.342436994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:22:24.34603137Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:22:24.566780076Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T12:22:27.064077987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:22:27.067191093Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:22:27.469058744Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:22:27.469092151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:22:27.469106471Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:22:27.469121653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:22:27.70505936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:22:27.706618246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T12:22:28.299101267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:22:28.302640033Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T12:22:28.921507431Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
ts=2026-02-16T12:22:29.333817297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-0285d981 10.128.0.61:7946"
level=info ts=2026-02-16T12:22:29.393328108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:22:29.396520902Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-02-16T12:22:29.620234001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:22:29.623326102Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:22:29.623345291Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:29.623365886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:22:29.625895782Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:22:29.705297172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:22:29.708487247Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:22:29.820426904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:22:29.823583743Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:22:29.823603729Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:29.923923936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:22:29.92706976Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T12:22:29.999354168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T12:22:30.343904795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41726"
level=warn ts=2026-02-16T12:22:32.46894187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:32.950839352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:22:32.957847652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:22:33.177530904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:22:33.180984465Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-02-16T12:22:33.276798663Z caller=checkpoint.go:378 msg="writing series" size="940 kB"
level=info ts=2026-02-16T12:22:35.804084127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:22:35.807236485Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T12:22:37.048850056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="35 kB" avg_comp="7.1 kB" total_uncomp="224 kB" avg_uncomp="45 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:22:37.052076731Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-02-16T12:22:37.468620815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:22:37.630947854Z caller=checkpoint.go:378 msg="writing series" size="1.4 kB"
level=info ts=2026-02-16T12:22:39.209088603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:22:39.212899153Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:22:39.244990243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="336 kB" avg_comp="112 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:22:39.24834173Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T12:22:40.230245752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:22:40.233589386Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T12:22:41.98622493Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=debug ts=2026-02-16T12:22:41.986250041Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-02-16T12:22:42.468998709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:42.603588554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:22:42.606887462Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:22:44.468061245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.6 kB" avg_comp="5.6 kB" total_uncomp="26 kB" avg_uncomp="26 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:22:44.47159424Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c6638380a%3A19c6638380b%3A17f91deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T12:22:44.471614953Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:44.471642297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="843 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:22:44.474519422Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
ts=2026-02-16T12:22:46.004766549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46078"
level=debug ts=2026-02-16T12:22:46.340271108Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-02-16T12:22:47.468482916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:49.462934426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:22:49.466416074Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T12:22:50.698953285Z caller=checkpoint.go:378 msg="writing series" size="9.4 MB"
level=debug ts=2026-02-16T12:22:50.698985894Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.4 MB" series=2
level=info ts=2026-02-16T12:22:52.34616742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="151 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:22:52.353512695Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c6638380a%3A19c66492190%3Aa67d8a38?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T12:22:52.353532143Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:52.353555226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:22:52.356768285Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T12:22:52.469263628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:53.988023809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:22:53.995261809Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:22:54.174166543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:22:54.177281429Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T12:22:55.050331772Z caller=checkpoint.go:378 msg="writing series" size="427 kB"
level=info ts=2026-02-16T12:22:56.299152117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:22:56.302559171Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T12:22:56.968512323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:22:56.971847571Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c6638380a%3A19c663883d8%3A18819bed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T12:22:56.971866432Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:56.971887539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="21 kB" avg_uncomp="21 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T12:22:56.974744222Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6638380a%3A19c665c28f0%3Aa13e7175?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T12:22:57.452636182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:22:57.453707551Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:22:57.468961467Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:22:57.468974076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:22:57.468981458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:22:57.468978294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:22:57.910867094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:22:57.914343505Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:22:57.914362524Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:22:57.914385495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:22:57.917090411Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
ts=2026-02-16T12:22:59.336590246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:22:59.405320223Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=info ts=2026-02-16T12:23:00.709487945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="35 kB" avg_comp="7.1 kB" total_uncomp="224 kB" avg_uncomp="45 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:23:00.712792441Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:23:01.775205319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:23:01.778522911Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T12:23:02.46876531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:02.568065132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:23:02.571264549Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=debug ts=2026-02-16T12:23:03.759839409Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-02-16T12:23:03.859207568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dda713edfcf30b5b immediate=false num_chunks=1 total_comp="6.6 kB" avg_comp="6.6 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"build-service\"}"
level=error ts=2026-02-16T12:23:03.863851932Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dda713edfcf30b5b/19c6638380a%3A19c664d1f7b%3Aead516f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-service\"}"
level=error ts=2026-02-16T12:23:03.863871907Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:23:03.863895264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.5 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:23:03.866620031Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=debug ts=2026-02-16T12:23:04.206592816Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:23:05.314720837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="523 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:23:05.317986933Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T12:23:05.998607569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:23:06.001790245Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=warn ts=2026-02-16T12:23:07.469062639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:23:08.115114728Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-02-16T12:23:08.209355231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:23:08.212829267Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:23:08.590204772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T12:23:08.593227868Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c6638380a%3A19c66386604%3A13d0313f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T12:23:11.428634735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:23:11.442478493Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:23:11.664400458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:23:11.667712701Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-02-16T12:23:12.46934684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:23:12.473622816Z caller=checkpoint.go:378 msg="writing series" size="2.1 MB"
level=debug ts=2026-02-16T12:23:12.473640835Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=5
level=info ts=2026-02-16T12:23:14.134525168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=4 total_comp="1.5 kB" avg_comp="367 B" total_uncomp="1.4 kB" avg_uncomp="342 B" full=4 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T12:23:14.138011877Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66406b5f%3A19c66406b60%3Aed3631a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T12:23:14.255715586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.2 kB" avg_comp="5.2 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T12:23:14.258934611Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6638380a%3A19c665c28f0%3Aa13e7175?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-02-16T12:23:15.578568432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52324"
level=debug ts=2026-02-16T12:23:16.824250723Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-02-16T12:23:17.468468623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:19.981551466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:23:19.985643888Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T12:23:20.964471684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=2 total_comp="13 kB" avg_comp="6.7 kB" total_uncomp="61 kB" avg_uncomp="30 kB" full=2 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-02-16T12:23:20.968299628Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b94b9dc89b8712c5/19c6637cfb9%3A19c663c3159%3Ada1b7672?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"tempo-operator\"}"
level=debug ts=2026-02-16T12:23:21.179339005Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=warn ts=2026-02-16T12:23:22.468456638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:22.905158562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:23:22.908205823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:23:22.908226366Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:23:22.908248763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:23:22.910916653Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T12:23:23.885396492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=4 total_comp="16 kB" avg_comp="4.1 kB" total_uncomp="158 kB" avg_uncomp="40 kB" full=4 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-02-16T12:23:23.888446164Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/973acd4420a50684/19c6637c959%3A19c66428c72%3Ad826e9e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-02-16T12:23:24.392922843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="564 B" avg_comp="564 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T12:23:24.396028657Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c6638380a%3A19c66386604%3A6e5191d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T12:23:24.984575637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:23:25.534257552Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-02-16T12:23:27.194227556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="411 B" avg_comp="411 B" total_uncomp="294 B" avg_uncomp="294 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T12:23:27.197667745Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c663ac82a%3A19c663ac82b%3A854bd03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T12:23:27.468585638Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:23:27.46861706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:23:27.46862595Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:23:27.468627904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:28.952313446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=3 total_comp="9.9 kB" avg_comp="3.3 kB" total_uncomp="46 kB" avg_uncomp="15 kB" full=3 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T12:23:28.955761333Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c6637ce57%3A19c66408103%3Aa9bbae86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T12:23:29.242385397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="637 B" avg_comp="637 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T12:23:29.24575134Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66384280%3A19c66384281%3A23db27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T12:23:29.340168151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:23:29.889430073Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-02-16T12:23:29.890472502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=3 total_comp="336 kB" avg_comp="112 kB" total_uncomp="3.3 MB" avg_uncomp="1.1 MB" full=3 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T12:23:29.894451496Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c6637d0f0%3A19c664731c0%3A464efb1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T12:23:30.348044485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55288"
level=info ts=2026-02-16T12:23:31.065314665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T12:23:31.072826864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c6637c93d%3A19c6639c507%3A58338fcf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T12:23:31.156155906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.7 kB" avg_comp="1.4 kB" total_uncomp="8.5 kB" avg_uncomp="4.2 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T12:23:31.159825481Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c6638380a%3A19c6638380b%3Ace119c06?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T12:23:32.346427578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.7 kB" avg_comp="852 B" total_uncomp="6.9 kB" avg_uncomp="3.5 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-02-16T12:23:32.34986767Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/daec03023a92ea81/19c6638380a%3A19c66386604%3A9978d2dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-02-16T12:23:32.468720028Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:33.609216274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="17 kB" avg_uncomp="8.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T12:23:33.612349774Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c6638380a%3A19c6638dbfb%3Af9876912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T12:23:33.94957611Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T12:23:34.090927533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T12:23:34.094497904Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c6638380a%3A19c66386604%3Ae383a408?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T12:23:34.243906132Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=info ts=2026-02-16T12:23:37.39970816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=2 total_comp="541 B" avg_comp="270 B" total_uncomp="545 B" avg_uncomp="272 B" full=2 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T12:23:37.403059554Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c6638380a%3A19c66386604%3Aed6c7fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T12:23:37.469017683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:23:38.599116266Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-02-16T12:23:42.46901066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T12:23:42.953849158Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
ts=2026-02-16T12:23:46.009981809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59392"
level=info ts=2026-02-16T12:23:47.058262989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=2 total_comp="1.1 kB" avg_comp="532 B" total_uncomp="2.5 kB" avg_uncomp="1.2 kB" full=2 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T12:23:47.063548324Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66386603%3A19c66386604%3Ac3b8744b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T12:23:47.308292771Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-02-16T12:23:47.468442005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:48.640935302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=2 total_comp="17 kB" avg_comp="8.6 kB" total_uncomp="96 kB" avg_uncomp="48 kB" full=2 labels="{stream=\"cert-manager\"}"
level=error ts=2026-02-16T12:23:48.644451189Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6bfb044b9a75b5b/19c6638380a%3A19c663906f2%3A2eeb63ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager\"}"
level=info ts=2026-02-16T12:23:49.553758134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="60 kB" avg_comp="60 kB" total_uncomp="340 kB" avg_uncomp="340 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T12:23:49.557404963Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c6637d5a6%3A19c663a58b0%3A6888e28b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T12:23:49.658846998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=5 total_comp="35 kB" avg_comp="7.1 kB" total_uncomp="224 kB" avg_uncomp="45 kB" full=5 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-02-16T12:23:49.662065875Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eab710ab81744b58/19c6637cb74%3A19c664a1bbf%3Ac775d0fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-02-16T12:23:50.963934906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T12:23:50.970508836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c6637ca10%3A19c665fccbb%3Aaeffc83e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T12:23:51.432208216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.2 kB" avg_comp="5.2 kB" total_uncomp="21 kB" avg_uncomp="21 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-02-16T12:23:51.435458407Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19c6638380a%3A19c665c28f0%3Aa13e7175?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-02-16T12:23:51.663495833Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-02-16T12:23:52.468574496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T12:23:54.306152381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:23:54.310933063Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:23:54.310952821Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:23:54.310982912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=2 total_comp="115 kB" avg_comp="58 kB" total_uncomp="1.0 MB" avg_uncomp="503 kB" full=2 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T12:23:54.313695106Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66384280%3A19c663904d8%3Ab3f4d237?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T12:23:54.8342924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:23:54.838274244Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:23:54.838294017Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T12:23:54.838318504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="525 kB" avg_comp="525 kB" total_uncomp="6.9 MB" avg_uncomp="6.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-02-16T12:23:54.841782021Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19c6637d783%3A19c665585a0%3A7f948299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-02-16T12:23:55.071645817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T12:23:55.07494047Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c6638380a%3A19c6639a8cf%3A143182a4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T12:23:56.018166676Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=debug ts=2026-02-16T12:23:57.452570935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.60:11211 count=1
level=info ts=2026-02-16T12:23:57.453627336Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T12:23:57.468913177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T12:23:57.468935137Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T12:23:57.468943647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T12:23:57.468991267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"